hf_trainer_mlp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hf_trainer_mlp.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5d93a746251b40ea8dedead0ef6660b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2bab92895f914f249b17a3161bf2dcf2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1e26d7bbfce74636bcdfb5d2430b6db6",
              "IPY_MODEL_563791a08b404263840ccca5c0c3d9d1",
              "IPY_MODEL_793b914105fd46468d5ac1fc23dbecb2"
            ]
          }
        },
        "2bab92895f914f249b17a3161bf2dcf2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1e26d7bbfce74636bcdfb5d2430b6db6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2f361df6056f445d9a4fbf344c8c2ab4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_876ca715e73e4215a1e38565a78d6fc6"
          }
        },
        "563791a08b404263840ccca5c0c3d9d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0485a28de5a44c01be017e1c0d1fc5ed",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a4fbf1d6e7df4d2da06677fd44dfa8c6"
          }
        },
        "793b914105fd46468d5ac1fc23dbecb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_af1fc13a5fc04353a746eaa550515e02",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00, 24.01it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c9aa9e152d734b4ab688c39fbe713fb5"
          }
        },
        "2f361df6056f445d9a4fbf344c8c2ab4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "876ca715e73e4215a1e38565a78d6fc6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0485a28de5a44c01be017e1c0d1fc5ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a4fbf1d6e7df4d2da06677fd44dfa8c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af1fc13a5fc04353a746eaa550515e02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c9aa9e152d734b4ab688c39fbe713fb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0cf22c88d6034d518e67ed8910fad15d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4e20db8eef12408ca33c9d0b0100b626",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_547d3269516b4cfcae3088cd7baef068",
              "IPY_MODEL_8a37f724a32b4cf59133743cf28d0808",
              "IPY_MODEL_84dfa52445654a9ebb6a964f3dc24302"
            ]
          }
        },
        "4e20db8eef12408ca33c9d0b0100b626": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "547d3269516b4cfcae3088cd7baef068": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_890d18c325d34ba48029260e34aaf6b3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9afcbf0d8dce4ae1ab7f0510eb1108e8"
          }
        },
        "8a37f724a32b4cf59133743cf28d0808": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7809404e10ce468185d5096a2490a300",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e806bc7740e4cb9af9f54bd1354800c"
          }
        },
        "84dfa52445654a9ebb6a964f3dc24302": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f6f5d17aab014329b2efceff5e3a3045",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00, 15.59it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a03d1a87bc0548a0a374df9d384ea0ce"
          }
        },
        "890d18c325d34ba48029260e34aaf6b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9afcbf0d8dce4ae1ab7f0510eb1108e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7809404e10ce468185d5096a2490a300": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e806bc7740e4cb9af9f54bd1354800c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f6f5d17aab014329b2efceff5e3a3045": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a03d1a87bc0548a0a374df9d384ea0ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7262beee45f249a1bc067f1732554772": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_17e7cbf7e4a2401ca3030f350acbf8aa",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4f7c7c3626134df08aa3c19ac46448e3",
              "IPY_MODEL_ff8c6d0d029f461691bdff7bc551b941",
              "IPY_MODEL_afac743ac17a4ae6bdfd0bedee72f11f"
            ]
          }
        },
        "17e7cbf7e4a2401ca3030f350acbf8aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f7c7c3626134df08aa3c19ac46448e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_633b8957a8e24958be761819311f84f4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f09dea8e14de4c5b80258d2ab098b2b9"
          }
        },
        "ff8c6d0d029f461691bdff7bc551b941": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4eb45cd6e995407d8bb5217a572ada09",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_43d16b152760449696774023d590ca64"
          }
        },
        "afac743ac17a4ae6bdfd0bedee72f11f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f86c65a052584d818db4481c7a8a8962",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [00:00&lt;00:00, 34.42it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bb32b2b1164740b5b04dce5d5b591ef6"
          }
        },
        "633b8957a8e24958be761819311f84f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f09dea8e14de4c5b80258d2ab098b2b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4eb45cd6e995407d8bb5217a572ada09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "43d16b152760449696774023d590ca64": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f86c65a052584d818db4481c7a8a8962": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bb32b2b1164740b5b04dce5d5b591ef6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a4ac5ab90ce49bf95d02bf8a385d68a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b5296d8548fe45cd9958ac2c5cb7d158",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_575665de34744518a5bd92bc28e33968",
              "IPY_MODEL_4275487240d44743890f1aad3d25218a",
              "IPY_MODEL_c90cd0467508468fbab85191c24533ce"
            ]
          }
        },
        "b5296d8548fe45cd9958ac2c5cb7d158": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "575665de34744518a5bd92bc28e33968": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1d0d3f71205041bfbc6a1dc7604e98d2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c20f24ce759b488c8e13df1149ddc1b5"
          }
        },
        "4275487240d44743890f1aad3d25218a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_59f970fb0712497ba8388af9ef8c8395",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ce85d702c20f44cb9fdb0a88afef75f4"
          }
        },
        "c90cd0467508468fbab85191c24533ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a8890d5e70944d57a847d941dcd7061a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1864/? [00:00&lt;00:00, 2769.72ex/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5350ba3fd87648c9a40712a77d9725ce"
          }
        },
        "1d0d3f71205041bfbc6a1dc7604e98d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c20f24ce759b488c8e13df1149ddc1b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "59f970fb0712497ba8388af9ef8c8395": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ce85d702c20f44cb9fdb0a88afef75f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a8890d5e70944d57a847d941dcd7061a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5350ba3fd87648c9a40712a77d9725ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8e23dd32574c456d9cdc637fb7a97a50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_67b56c5ac1e44f9baac01277602d09b9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_84e9a81c6596408895a747226e26e4f7",
              "IPY_MODEL_de78b17d5aac471b884e9bed29f5da2d",
              "IPY_MODEL_248eabeb5f5e44beb21f4d6086c9bf9c"
            ]
          }
        },
        "67b56c5ac1e44f9baac01277602d09b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "84e9a81c6596408895a747226e26e4f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_41d58d96046642ffb3cfd6a0c1094d06",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_465e8a6273934edcad310d51abdfcb1a"
          }
        },
        "de78b17d5aac471b884e9bed29f5da2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0033f91629e74adfbee769c12348dee6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0f2f64195dfb45e394c6607cc7948a66"
          }
        },
        "248eabeb5f5e44beb21f4d6086c9bf9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_aae5c8077ca042aaa22aa431172844f3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 233/? [00:00&lt;00:00, 1108.51ex/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_21002e687a364965bca327cc153b0c6c"
          }
        },
        "41d58d96046642ffb3cfd6a0c1094d06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "465e8a6273934edcad310d51abdfcb1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0033f91629e74adfbee769c12348dee6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0f2f64195dfb45e394c6607cc7948a66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aae5c8077ca042aaa22aa431172844f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "21002e687a364965bca327cc153b0c6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a050280a80a44399956faa4b882b64c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d3198f086786427eae7bfd860d76057a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_927b75827d0e44c39fae83f54b112cf5",
              "IPY_MODEL_0053b3484bba4073a264147162645704",
              "IPY_MODEL_7831dec1d4f44a1c99b7b2af7d44a7cd"
            ]
          }
        },
        "d3198f086786427eae7bfd860d76057a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "927b75827d0e44c39fae83f54b112cf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4d32ead1927442829371a2827013f61a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f56d83e17e654659ade6a5542e378fff"
          }
        },
        "0053b3484bba4073a264147162645704": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1171d95fc80e41559a4a41cf1933c146",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8690e5cc9e78434f9d8175da020bfa88"
          }
        },
        "7831dec1d4f44a1c99b7b2af7d44a7cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e9df72508bc944f6a88847ed1e52256e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 233/? [00:00&lt;00:00, 1155.35ex/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cb9b6c39230a44bb883cc0817b9610b2"
          }
        },
        "4d32ead1927442829371a2827013f61a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f56d83e17e654659ade6a5542e378fff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1171d95fc80e41559a4a41cf1933c146": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8690e5cc9e78434f9d8175da020bfa88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e9df72508bc944f6a88847ed1e52256e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cb9b6c39230a44bb883cc0817b9610b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a88a06916854c41a9f7c18fd9357bd7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a97123558cb84ad28001f61fca39bfa6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3d9bc3d7e65f426891c23434043c3715",
              "IPY_MODEL_e18391811d524aa0b16429e7638c5b7c",
              "IPY_MODEL_2df16c03e67d40388f77e457f69937c3"
            ]
          }
        },
        "a97123558cb84ad28001f61fca39bfa6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d9bc3d7e65f426891c23434043c3715": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2e8352462eb14cfda0e3672aa4549708",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: ",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a94285a77d034c64bfc33798482a6958"
          }
        },
        "e18391811d524aa0b16429e7638c5b7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1b2608cb41de474cbdd1c65c163ad02e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1411,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1411,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7a3b26aff8a74288bd959aedfe962ede"
          }
        },
        "2df16c03e67d40388f77e457f69937c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_653f076ec50f424e99b70ff48d6607ee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3.19k/? [00:00&lt;00:00, 81.3kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cfd9310f092941a883b060bcccb85f80"
          }
        },
        "2e8352462eb14cfda0e3672aa4549708": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a94285a77d034c64bfc33798482a6958": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1b2608cb41de474cbdd1c65c163ad02e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7a3b26aff8a74288bd959aedfe962ede": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "653f076ec50f424e99b70ff48d6607ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cfd9310f092941a883b060bcccb85f80": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/Deep_Learning_in_LangTech_course/blob/master/hf_trainer_mlp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setup\n",
        "\n",
        "Before we start running our own Python code, install the required Python packages using [pip](https://en.wikipedia.org/wiki/Pip):\n",
        "\n",
        "* [`transformers`](https://huggingface.co/docs/transformers/index) is a popular deep learning package\n",
        "* [`datasets`](https://huggingface.co/docs/datasets/) provides support for loading, creating, and manipulating datasets\n",
        "\n",
        "Both of these packages are used extensively on this course."
      ],
      "metadata": {
        "id": "LDZZUKzfPRWT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install -q transformers datasets"
      ],
      "metadata": {
        "id": "fKPHBYptQDsK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33d14c57-199a-48fe-e0df-e0040044a2fc"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 3.8 MB 5.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 312 kB 51.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 29.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 895 kB 43.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 596 kB 48.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 67 kB 4.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 212 kB 47.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 134 kB 46.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 42.0 MB/s \n",
            "\u001b[K     |████████████████████████████████| 127 kB 30.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 271 kB 41.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 94 kB 3.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 144 kB 47.1 MB/s \n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "(Above, the `!` at the start of the line tells the notebook to run the line as an operating system command rather than Python code, and the `-q` argument to `pip` runs the command in \"quiet\" mode, with less output.)"
      ],
      "metadata": {
        "id": "xOeI-LA9RcYL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll also use the [`pprint`](https://docs.python.org/3/library/pprint.html) (\"pretty-print\") module to format output more readably below. The only difference to just using `print` is that some data structures will be easier to read and interpret."
      ],
      "metadata": {
        "id": "i92pgm64t2w5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pprint import PrettyPrinter\n",
        "\n",
        "pprint = PrettyPrinter(compact=True).pprint"
      ],
      "metadata": {
        "id": "-aN2R9OxuRUb"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Finally, you can reduce logging output. The `transformers` library by default produces fairly verbose logging. Uncommenting the following code will disable low-priority output (`INFO` logging level and below)."
      ],
      "metadata": {
        "id": "5bIOCYhWYL1j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import logging\n",
        "\n",
        "#logging.disable(logging.INFO)"
      ],
      "metadata": {
        "id": "0auWk_z4YcVo"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "# Download and prepare data\n",
        "\n",
        "The data used in this notebook is found online in as a single file in a [tab-separated values](https://en.wikipedia.org/wiki/Tab-separated_values) (TSV) format. We first use the [`wget`](https://en.wikipedia.org/wiki/Wget) operating system command to download the data to the computer running this notebook."
      ],
      "metadata": {
        "id": "_fCdfQfNNzwt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "h-QSkx0O2sMK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "de7af550-9f0f-4450-c157-e65b048916f1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File ‘sentiment-target-fi.tsv’ already there; not retrieving.\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget -nc https://raw.githubusercontent.com/TurkuNLP/sentiment-target-corpus/main/sentiment-target-fi.tsv"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "(Above, the `-nc` argument *tells* `wget` to run in \"quiet\" mode and not to download the file again if it is already found on the computer. The latter option is useful in particular if running a notebook more than once.)"
      ],
      "metadata": {
        "id": "x3R3WzwvSIO6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's have a look at the first few lines of the file to get an idea of its contents."
      ],
      "metadata": {
        "id": "nSsSdZ46S_Yb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "lines = []\n",
        "with open(\"sentiment-target-fi.tsv\") as f:    # open file\n",
        "    for line in f:    # iterate over lines in file\n",
        "        line = line.rstrip(\"\\n\")    # remove newline character\n",
        "        lines.append(line)    # store line in list\n",
        "\n",
        "for line in lines[:10]:    # iterate over first ten lines\n",
        "    print(line)    # print each line"
      ],
      "metadata": {
        "id": "QxmgHoKDTN2_",
        "outputId": "f29a73bf-15e9-4250-8aa5-a37f34eec840",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "#ID\ttext\tmajority\tagreement\tannotations\n",
            "comments2015d-45552-1.1\t<TARGET>Kuopionlahden</TARGET> vierasvenelaituri (kuten nimi oikeasti on) on mielestämme tosi kiva.\tpositive\t0.6666666666666666\tpositive,positive,neither,positive,neither,positive\n",
            "comments2014a-469816-3.1\tKäytetyssä <TARGET>Meganessa</TARGET> yksi etu verrattuna vaikkapa Skoda Octaviaan on se että Meganen voi ostaa jonkun vuoden uudempana.\tpositive\t1.0\tpositive,positive,positive,positive,positive,positive\n",
            "comments2007d-182055-6.1\tMonet heistä ovat olleet kymmeniä vuosia <TARGET>Finnairin</TARGET> palveluksessa ja nyt tulee kenkää.\tnegative\t0.8\tnegative,negative,negative,neither,negative\n",
            "threads2013a-835251-2.1\t<TARGET>Keskusta</TARGET> on aina ollut (ja on oleva) muun Suomen tuki ja turva.\tpositive\t1.0\tpositive,positive,positive\n",
            "comments2009a-483243-7.1\tItselläni on vuosimallin -93 <TARGET>LS-400</TARGET> ja olen tyytyväinen.\tpositive\t1.0\tpositive,positive,positive,positive,positive\n",
            "comments2003a-590164-4.1\t<TARGET>The Beatles</TARGET> on edelleenki maailma parasta musiikkia, eikä sitä voita mikään!\tpositive\t1.0\tpositive,positive,positive,positive,positive\n",
            "threads2011b-327878-1.2\tRoskaruokaa mukaan.. ... myyville (Mäcit, <TARGET>Heset</TARGET> ymv.) pitäisi laittaa jäte-/roskaamismaksu.\tnegative\t0.6666666666666666\tnegative,positive,negative,negative,negative,neither\n",
            "comments2014d-822316-1.1\tJos jotain pitäisi alkaa tutkia, on se <TARGET>rfsu:n</TARGET> motiivit ilmoittaa julkisesti selvästi todellisuutta pienempiä lukuja.\tnegative\t1.0\tnegative,negative,negative\n",
            "comments2005c-69174-2.2\tVene ei ole Wellu, ihan <TARGET>Yamarin</TARGET> -merkkinen vaan, mutta kohtuu nopea kuitenkin (n.38 s.)\tpositive\t0.5\tpositive,neither\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We see above that the first line is a \"header\" starting with the `#` character used for comments in Python code and the lines contain five fields separated by TAB characters:\n",
        "\n",
        "* `ID`: unique identifier (not required here)\n",
        "* `text`: the text of the example, with sentiment target marked with `<TARGET>...</TARGET>`\n",
        "* `majority`: the label assigned to the example by the majority of annotators\n",
        "* `agreement`: the ratio of all annotators that assigned the majority label\n",
        "* `annotations`: a comma-separated list of all of the annotations for the example"
      ],
      "metadata": {
        "id": "NNKIR8drYUxn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reformat data for `datasets` package\n",
        "\n",
        "* You really only need to do this once\n",
        "* Make a [*json lines*](https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON) (JSONL) file with one [JSON](https://en.wikipedia.org/wiki/JSON)-encoded example per line\n",
        "* Each example has the `text` and the `label` as an integer\n",
        "* We have four different labels in this particular data"
      ],
      "metadata": {
        "id": "7F7MWI_TKoC9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import json\n",
        "\n",
        "label_names=[\"positive\",\"negative\",\"reject\",\"neither\"]\n",
        "data=[]\n",
        "with open(\"sentiment-target-fi.tsv\") as f:\n",
        "    for line in f:\n",
        "        line=line.rstrip(\"\\n\")\n",
        "        if not line or line.startswith(\"#\"): #skip empty and comments (incl. header)\n",
        "            continue\n",
        "        cols=line.split(\"\\t\")\n",
        "        if len(cols)!=5: #skip weird lines that don't have the right number of columns\n",
        "            continue\n",
        "        data.append(cols)"
      ],
      "metadata": {
        "id": "6WZKk7xW3OMu"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll randomly shuffle the data to assure that our split into training, validation and test data is random.\n",
        "\n",
        "Note that here we call [`random.seed`](https://docs.python.org/3/library/random.html#bookkeeping-functions) with an arbitrary integer value so that the shuffle always results in the same split. This is not necessary (and must be avoided if you e.g. need different shuffles!), but useful here as it will make results in this notebook more stable. "
      ],
      "metadata": {
        "id": "JmUkdxA1vUBS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "random.seed(1234)    # seed with arbitrary integer value\n",
        "random.shuffle(data) # produces arbitrary but fixed order"
      ],
      "metadata": {
        "id": "rNxvxKe7wWBx"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, let's cast the data into simple python dictionaries and save these as JSONL."
      ],
      "metadata": {
        "id": "80gyeQuIwnNx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"sentiment-data.jsonl\",\"wt\") as f:\n",
        "    for cols in data:\n",
        "        item = {\n",
        "            \"text\": cols[1],\n",
        "            \"label\": label_names.index(cols[2]),    # translate from label strings to integers\n",
        "        }\n",
        "        print(json.dumps(item,ensure_ascii=False,sort_keys=True),file=f)"
      ],
      "metadata": {
        "id": "OJi4jsAtwivq"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's read back the file we just created and print one example to make sure it looks as intended:"
      ],
      "metadata": {
        "id": "Zsa_uNQJaSqF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "examples = []\n",
        "with open(\"sentiment-data.jsonl\") as f:\n",
        "    for line in f:\n",
        "        example = json.loads(line)\n",
        "        examples.append(example)\n",
        "\n",
        "pprint(examples[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OA_uAKT9a3Vs",
        "outputId": "cbcd0b35-42fe-4be2-8453-33581c5725ec"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'label': 0,\n",
            " 'text': 'Mä pidän eniten <TARGET>Blessed Be</TARGET>-levystä, joten varsinkin '\n",
            "         'ne biisit olivat mun mieleen!'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load data with `datasets`\n",
        "\n",
        "Every popular framework has its own preferred idea of how to represent data. We will here use the Hugging Face [datasets](https://huggingface.co/docs/datasets/) library, which is very popular, so it makes sense to be acquainted with it.\n",
        "\n",
        "The `datasets` library provides broad support for [loading datasets](https://huggingface.co/docs/datasets/loading_datasets.html) through the `load_dataset` function, which we will use here to load the JSONL-formatted data we prepared just above."
      ],
      "metadata": {
        "id": "r7oHDRn7LgW2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import datasets\n",
        "\n",
        "file_name = \"sentiment-data.jsonl\"\n",
        "dataset = datasets.load_dataset(\n",
        "    'json',                             # Format of the data\n",
        "    data_files={\"everything\":file_name},    # All data files, here we only have one\n",
        "    split={\n",
        "        \"train\":\"everything[:80%]\",  # First 80% is the train set\n",
        "        \"validation\":\"everything[80%:90%]\",   # Next 10% is the validation/dev set\n",
        "        \"test\":\"everything[90%:]\"    # last 10% is the test set\n",
        "    },\n",
        "    features=datasets.Features({    # Here we tell how to interpret the attributes\n",
        "        \"label\":datasets.ClassLabel(names=label_names),\n",
        "        \"text\":datasets.Value(\"string\")\n",
        "    })\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185,
          "referenced_widgets": [
            "5d93a746251b40ea8dedead0ef6660b4",
            "2bab92895f914f249b17a3161bf2dcf2",
            "1e26d7bbfce74636bcdfb5d2430b6db6",
            "563791a08b404263840ccca5c0c3d9d1",
            "793b914105fd46468d5ac1fc23dbecb2",
            "2f361df6056f445d9a4fbf344c8c2ab4",
            "876ca715e73e4215a1e38565a78d6fc6",
            "0485a28de5a44c01be017e1c0d1fc5ed",
            "a4fbf1d6e7df4d2da06677fd44dfa8c6",
            "af1fc13a5fc04353a746eaa550515e02",
            "c9aa9e152d734b4ab688c39fbe713fb5",
            "0cf22c88d6034d518e67ed8910fad15d",
            "4e20db8eef12408ca33c9d0b0100b626",
            "547d3269516b4cfcae3088cd7baef068",
            "8a37f724a32b4cf59133743cf28d0808",
            "84dfa52445654a9ebb6a964f3dc24302",
            "890d18c325d34ba48029260e34aaf6b3",
            "9afcbf0d8dce4ae1ab7f0510eb1108e8",
            "7809404e10ce468185d5096a2490a300",
            "7e806bc7740e4cb9af9f54bd1354800c",
            "f6f5d17aab014329b2efceff5e3a3045",
            "a03d1a87bc0548a0a374df9d384ea0ce",
            "7262beee45f249a1bc067f1732554772",
            "17e7cbf7e4a2401ca3030f350acbf8aa",
            "4f7c7c3626134df08aa3c19ac46448e3",
            "ff8c6d0d029f461691bdff7bc551b941",
            "afac743ac17a4ae6bdfd0bedee72f11f",
            "633b8957a8e24958be761819311f84f4",
            "f09dea8e14de4c5b80258d2ab098b2b9",
            "4eb45cd6e995407d8bb5217a572ada09",
            "43d16b152760449696774023d590ca64",
            "f86c65a052584d818db4481c7a8a8962",
            "bb32b2b1164740b5b04dce5d5b591ef6"
          ]
        },
        "id": "dvu2uxrk6_YW",
        "outputId": "610f8a57-8945-43be-cfca-12624e15a26d"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Using custom data configuration default-a5d61dd4920e4638\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset json/default to /root/.cache/huggingface/datasets/json/default-a5d61dd4920e4638/0.0.0/ac0ca5f5289a6cf108e706efcf040422dbbfa8e658dee6a819f20d76bb84d26b...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5d93a746251b40ea8dedead0ef6660b4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0cf22c88d6034d518e67ed8910fad15d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset json downloaded and prepared to /root/.cache/huggingface/datasets/json/default-a5d61dd4920e4638/0.0.0/ac0ca5f5289a6cf108e706efcf040422dbbfa8e658dee6a819f20d76bb84d26b. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7262beee45f249a1bc067f1732554772",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's see what's in the dataset that we just loaded:"
      ],
      "metadata": {
        "id": "qnq8vbZUqPaN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "itvflIheqO-A",
        "outputId": "cd279fe6-a220-451b-9d45-2a4ff1bab4b0"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['label', 'text'],\n",
            "        num_rows: 1864\n",
            "    })\n",
            "    validation: Dataset({\n",
            "        features: ['label', 'text'],\n",
            "        num_rows: 233\n",
            "    })\n",
            "    test: Dataset({\n",
            "        features: ['label', 'text'],\n",
            "        num_rows: 233\n",
            "    })\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As intended, we see here a datasets in three parts, `train`, `validation`, and `test`, where each has the `text` and `label` values.\n",
        "\n",
        "We can look at individual values e.g. as follows:"
      ],
      "metadata": {
        "id": "nijUhsWarax3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(dataset['train'][0]['text'])\n",
        "print(dataset['train'][0]['label'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KI3kyD3vry8C",
        "outputId": "67bd7651-f610-405f-88bf-7f5623015b9b"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mä pidän eniten <TARGET>Blessed Be</TARGET>-levystä, joten varsinkin ne biisit olivat mun mieleen!\n",
            "0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenize and vectorize data\n",
        "\n",
        "Each text in our dataset is a character string. As you know, machine learning methods do not generally operate directly on character strings. Instead, text is typically _tokenized_ and _vectorized_ for input to machine learning methods.\n",
        "\n",
        "Here, we will use previously created tokenizers through the simple [`AutoTokenizer`](https://huggingface.co/docs/transformers/model_doc/auto#transformers.AutoTokenizer) class. The `AutoTokenizer.from_pretrained` function can load the tokenizer associated with any of the large number of models found in the [Hugging Face models repository](https://huggingface.co/models).            "
      ],
      "metadata": {
        "id": "KMnqQ78cMpk2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import transformers\n",
        "\n",
        "model_name = \"TurkuNLP/bert-base-finnish-cased-v1\"\n",
        "tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)"
      ],
      "metadata": {
        "id": "MYOtKJu7Mohd"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The tokenizers in the `transformers` library take a text string as argument and both split the text into tokens (tokenization) and map those tokens into integer identifiers (vectorization).\n",
        "\n",
        "The values returned by the tokenizer depend on the model that the tokenizer was created for, but normally include at least `input_ids`, which are the result of tokenization and vectorization:"
      ],
      "metadata": {
        "id": "jjqi3BttlW8Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(tokenizer(\"Tämä on esimerkkilause\"))\n",
        "tokenizer.tokenize(\"Tämä on esimerkkilause\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RwB5pZWZo4qg",
        "outputId": "fcfc0837-343e-481a-e42e-a00e0f50a623"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'attention_mask': [1, 1, 1, 1, 1, 1, 1],\n",
            " 'input_ids': [102, 1131, 145, 6755, 6917, 50010, 103],\n",
            " 'token_type_ids': [0, 0, 0, 0, 0, 0, 0]}\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Tämä', 'on', 'esimerkki', '##laus', '##e']"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can ignore the values other than `input_ids` returned by the tokenizer (`token_type_ids` and `attention_mask`) for now. We will come back to these later in the course."
      ],
      "metadata": {
        "id": "FN6VIC9EpHeg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can now tokenize and vectorize our whole dataset with the tokenizer by calling the [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) function:"
      ],
      "metadata": {
        "id": "zLyNGe7TpqJo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a simple function that applies the tokenizer\n",
        "def tokenize(example):\n",
        "    return tokenizer(example[\"text\"])\n",
        "\n",
        "# Apply the tokenizer to the whole dataset using .map()\n",
        "dataset = dataset.map(tokenize)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "1a4ac5ab90ce49bf95d02bf8a385d68a",
            "b5296d8548fe45cd9958ac2c5cb7d158",
            "575665de34744518a5bd92bc28e33968",
            "4275487240d44743890f1aad3d25218a",
            "c90cd0467508468fbab85191c24533ce",
            "1d0d3f71205041bfbc6a1dc7604e98d2",
            "c20f24ce759b488c8e13df1149ddc1b5",
            "59f970fb0712497ba8388af9ef8c8395",
            "ce85d702c20f44cb9fdb0a88afef75f4",
            "a8890d5e70944d57a847d941dcd7061a",
            "5350ba3fd87648c9a40712a77d9725ce",
            "8e23dd32574c456d9cdc637fb7a97a50",
            "67b56c5ac1e44f9baac01277602d09b9",
            "84e9a81c6596408895a747226e26e4f7",
            "de78b17d5aac471b884e9bed29f5da2d",
            "248eabeb5f5e44beb21f4d6086c9bf9c",
            "41d58d96046642ffb3cfd6a0c1094d06",
            "465e8a6273934edcad310d51abdfcb1a",
            "0033f91629e74adfbee769c12348dee6",
            "0f2f64195dfb45e394c6607cc7948a66",
            "aae5c8077ca042aaa22aa431172844f3",
            "21002e687a364965bca327cc153b0c6c",
            "a050280a80a44399956faa4b882b64c6",
            "d3198f086786427eae7bfd860d76057a",
            "927b75827d0e44c39fae83f54b112cf5",
            "0053b3484bba4073a264147162645704",
            "7831dec1d4f44a1c99b7b2af7d44a7cd",
            "4d32ead1927442829371a2827013f61a",
            "f56d83e17e654659ade6a5542e378fff",
            "1171d95fc80e41559a4a41cf1933c146",
            "8690e5cc9e78434f9d8175da020bfa88",
            "e9df72508bc944f6a88847ed1e52256e",
            "cb9b6c39230a44bb883cc0817b9610b2"
          ]
        },
        "id": "33xMYRd0q2B9",
        "outputId": "ec4b5e63-fb09-41a5-d905-e2b945329a72"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1a4ac5ab90ce49bf95d02bf8a385d68a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0ex [00:00, ?ex/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8e23dd32574c456d9cdc637fb7a97a50",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0ex [00:00, ?ex/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a050280a80a44399956faa4b882b64c6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0ex [00:00, ?ex/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Note**: unlike the Python`map` function, [`Dataset.map`](https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map) function _updates_ its argument dataset, keeping existing values. Here, the call adds the values returned by the `tokenizer()` call (`input_ids` etc.) to each example while also keeping the original `text` and `label` values."
      ],
      "metadata": {
        "id": "KJY01ZgodGTp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pprint(dataset[\"train\"][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pcrUMb0ewBZ_",
        "outputId": "bc0559ef-d25b-4f30-8b9d-a3dc21d0abd6"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "                    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
            " 'input_ids': [102, 2175, 6623, 3592, 5571, 16307, 50051, 50073, 12355, 2377,\n",
            "               22145, 1296, 369, 9627, 5571, 499, 16307, 50051, 50073, 12355,\n",
            "               2377, 166, 5390, 492, 119, 861, 3213, 475, 33571, 1141, 1662,\n",
            "               3063, 380, 103],\n",
            " 'label': 0,\n",
            " 'text': 'Mä pidän eniten <TARGET>Blessed Be</TARGET>-levystä, joten varsinkin '\n",
            "         'ne biisit olivat mun mieleen!',\n",
            " 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "                    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Input encoding for MLP\n",
        "\n",
        "Out `input_ids` are currently represented as a sparse vector containing the indices of the tokens found in the text (see e.g. [Torch documentation](https://pytorch.org/docs/stable/sparse.html)).\n",
        "\n",
        "The simplest way to present this input to the MLP model is to create a dense \"bag-of-tokens\" representation where the index for every token that is present in the input is 1, and the rest are 0.\n",
        "\n",
        "For example, if our input has vocab size of 5 and tokens `[0,3]` are present, we would like to get `[1,0,0,1,0]` on the input. The simple code below does just that; for the documentation, see [`torch.Tensor.scatter()`](https://pytorch.org/docs/stable/generated/torch.Tensor.scatter.html)"
      ],
      "metadata": {
        "id": "XTtyHQpIIJWw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "# Create a matrix of two examples (vectors) with\n",
        "# the ids (indices) which we want to set to 1\n",
        "input_ids = torch.tensor(\n",
        "  [\n",
        "     [0,0,1],\n",
        "     [0,2,3]\n",
        "  ]\n",
        ")\n",
        "\n",
        "vocab_size = 5\n",
        "example_num = len(input_ids)\n",
        "\n",
        "# These are the 1s we will be copying over\n",
        "ones = torch.ones_like(input_ids, dtype=torch.float)\n",
        "\n",
        "# This is the target, initialized to zeros\n",
        "zeros = torch.zeros((example_num, vocab_size))\n",
        "\n",
        "# Scatter says: \n",
        "#   work on dimension 1\n",
        "#   `input_ids` are the indices to set\n",
        "#   `ones` are the values to set\n",
        "input = zeros.scatter(1, input_ids, ones)\n",
        "\n",
        "print(input)\n",
        "# see how in the first row indices 0 and 1 are set to 1\n",
        "# and in the second row indices 0,2,3 are set to 1\n",
        "# exactly as it was supposed to be!"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DKPChR9OeRcb",
        "outputId": "f16b1be2-4a34-493e-b4da-26b352c618a0"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[1., 1., 0., 0., 0.],\n",
            "        [1., 0., 1., 1., 0.]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Build model\n",
        "\n",
        "A PyTorch model class in its simplest form has `__init__()` which instantiates the layers and `forward()` which implements the actual computation. For more information on these, please see the [PyTorch turorial](https://pytorch.org/tutorials/beginner/introyt/modelsyt_tutorial.html)."
      ],
      "metadata": {
        "id": "AOYYF5I1OWG9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "# A model wants a config, I can simply inherit from the base\n",
        "# class for pretrained configs\n",
        "class MLPConfig(transformers.PretrainedConfig):\n",
        "    pass\n",
        "\n",
        "# This is the model\n",
        "class MLP(transformers.PreTrainedModel):\n",
        "\n",
        "    config_class=MLPConfig\n",
        "\n",
        "    # In the initialization method, one instantiates the layers\n",
        "    # these will be the parameters of the model\n",
        "    def __init__(self,config):\n",
        "        super().__init__(config)\n",
        "        self.vocab_size=config.vocab_size\n",
        "        # Hidden layer: input size x hidden size\n",
        "        self.hidden=torch.nn.Linear(in_features=self.vocab_size,out_features=config.hidden_size)\n",
        "        # Output layer: hidden size x output size\n",
        "        self.output=torch.nn.Linear(in_features=config.hidden_size,out_features=config.nlabels)\n",
        "        \n",
        "    # The computation of the model is put into the forward() function\n",
        "    # it receives a batch of data and optionally the correct `labels`\n",
        "    #\n",
        "    # If given `labels` it returns (loss,output)\n",
        "    # if not, then it returns (output,)\n",
        "    def forward(self,input_ids,labels=None,attention_mask=None): #nevermind the attention_mask, its time will come, data collator insists on adding it\n",
        "        # The batch is in input_ids\n",
        "        batch_size=input_ids.shape[0] #this is how many examples we have\n",
        "        # The following block converts the input ids into a suitable input for\n",
        "        # the input layer, it is adapted from above\n",
        "        input=torch.zeros((batch_size,self.vocab_size),dtype=torch.float,device=input_ids.device)\n",
        "        ones=torch.ones_like(input_ids,dtype=torch.float)\n",
        "        input=input.scatter(1,input_ids,ones)\n",
        "        projected=torch.tanh(self.hidden(input)) #Note how non-linearity is applied here and not when configuring the layer in __init__()\n",
        "        logits=self.output(projected)\n",
        "        \n",
        "        # We have labels, so we ought to calculate the loss\n",
        "        if labels is not None:\n",
        "            loss=torch.nn.CrossEntropyLoss() #This loss is useful for classification\n",
        "            # You run it as loss(model_output,correct_labels)\n",
        "            return (loss(logits,labels),logits)\n",
        "        else:\n",
        "            # No labels, so just return the logits\n",
        "            return (logits,)\n",
        "\n",
        "# Configure the model:\n",
        "#   these parameters are used in the model's __init__()\n",
        "mlp_config=MLPConfig(vocab_size=tokenizer.vocab_size,hidden_size=100,nlabels=4)\n"
      ],
      "metadata": {
        "id": "vP4FtrMwCpGi"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train model\n",
        "\n",
        "We will use the Hugging Face [Trainer](https://huggingface.co/docs/transformers/main_classes/trainer) class for training\n",
        "\n",
        "* Loads of arguments that control the training\n",
        "* Configurable metrics to evaluate performance\n",
        "* Data collator builds the batches\n",
        "* Early stopping callback stops when eval loss no longer improves\n",
        "  "
      ],
      "metadata": {
        "id": "tdlcMObzQGGP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, let's create a [`TrainingArguments`](https://huggingface.co/docs/transformers/v4.17.0/en/main_classes/trainer#transformers.TrainingArguments) object to specify hyperparameters and various other settings for training. \n",
        "\n",
        "Printing this simple dataclass object will show not only the values we set, but also the defaults for all other arguments. Don't worry if you don't understand what all of these do! Many are not relevant to us here, and you can find the details in [`Trainer` documentation](https://huggingface.co/docs/transformers/main_classes/trainer) if you are interested."
      ],
      "metadata": {
        "id": "aZgcNi4B76SX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set training arguments\n",
        "trainer_args = transformers.TrainingArguments(\n",
        "    \"mlp_checkpoints\",\n",
        "    evaluation_strategy=\"steps\",\n",
        "    logging_strategy=\"steps\",\n",
        "    eval_steps=100,\n",
        "    logging_steps=100,\n",
        "    learning_rate=5e-4,\n",
        "    max_steps=5000,\n",
        "    load_best_model_at_end=True\n",
        ")\n",
        "\n",
        "print(trainer_args)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IhfdW62z8cCn",
        "outputId": "7f1057b7-66c6-4397-e238-40673265a87a"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TrainingArguments(\n",
            "_n_gpu=1,\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "bf16=False,\n",
            "bf16_full_eval=False,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_pin_memory=True,\n",
            "ddp_bucket_cap_mb=None,\n",
            "ddp_find_unused_parameters=None,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "do_eval=True,\n",
            "do_predict=False,\n",
            "do_train=False,\n",
            "eval_accumulation_steps=None,\n",
            "eval_steps=100,\n",
            "evaluation_strategy=IntervalStrategy.STEPS,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "gradient_accumulation_steps=1,\n",
            "gradient_checkpointing=False,\n",
            "greater_is_better=False,\n",
            "group_by_length=False,\n",
            "half_precision_backend=auto,\n",
            "hub_model_id=None,\n",
            "hub_strategy=HubStrategy.EVERY_SAVE,\n",
            "hub_token=<HUB_TOKEN>,\n",
            "ignore_data_skip=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=0.0005,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=True,\n",
            "local_rank=-1,\n",
            "log_level=-1,\n",
            "log_level_replica=-1,\n",
            "log_on_each_node=True,\n",
            "logging_dir=mlp_checkpoints/runs/Mar10_12-24-56_4e3fa3659b4a,\n",
            "logging_first_step=False,\n",
            "logging_nan_inf_filter=True,\n",
            "logging_steps=100,\n",
            "logging_strategy=IntervalStrategy.STEPS,\n",
            "lr_scheduler_type=SchedulerType.LINEAR,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=5000,\n",
            "metric_for_best_model=loss,\n",
            "mp_parameters=,\n",
            "no_cuda=False,\n",
            "num_train_epochs=3.0,\n",
            "optim=OptimizerNames.ADAMW_HF,\n",
            "output_dir=mlp_checkpoints,\n",
            "overwrite_output_dir=False,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=8,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=None,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=mlp_checkpoints,\n",
            "save_on_each_node=False,\n",
            "save_steps=500,\n",
            "save_strategy=IntervalStrategy.STEPS,\n",
            "save_total_limit=None,\n",
            "seed=42,\n",
            "sharded_ddp=[],\n",
            "skip_memory_metrics=True,\n",
            "tf32=None,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_legacy_prediction_loop=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            "xpu_backend=None,\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next, let's create a metric for evaluating performance during and after training. We can use the convenience function [`load_metric`](https://huggingface.co/docs/datasets/about_metrics) to load one of many pre-made metrics and wrap this for use by the trainer.\n",
        "\n",
        "As the task is simple multi-class classification, we'll use the basic `accuracy` metric, defined as the proportion of correctly predicted labels out of all labels."
      ],
      "metadata": {
        "id": "4sJwNXPU-dQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "accuracy = datasets.load_metric(\"accuracy\")\n",
        "\n",
        "def compute_accuracy(outputs_and_labels):\n",
        "    outputs, labels = outputs_and_labels\n",
        "    predictions = np.argmax(outputs, axis=-1) #pick the index of the \"winning\" label\n",
        "    return accuracy.compute(predictions=predictions, references=labels)"
      ],
      "metadata": {
        "id": "u3jxIItb0BL9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "9a88a06916854c41a9f7c18fd9357bd7",
            "a97123558cb84ad28001f61fca39bfa6",
            "3d9bc3d7e65f426891c23434043c3715",
            "e18391811d524aa0b16429e7638c5b7c",
            "2df16c03e67d40388f77e457f69937c3",
            "2e8352462eb14cfda0e3672aa4549708",
            "a94285a77d034c64bfc33798482a6958",
            "1b2608cb41de474cbdd1c65c163ad02e",
            "7a3b26aff8a74288bd959aedfe962ede",
            "653f076ec50f424e99b70ff48d6607ee",
            "cfd9310f092941a883b060bcccb85f80"
          ]
        },
        "outputId": "faaa8821-42fe-4449-ac2a-f89c32af203b"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a88a06916854c41a9f7c18fd9357bd7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.41k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then create the `Trainer` and train the model by invoking the [`Trainer.train`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.train) function.\n",
        "\n",
        "In addition to the model, the settings passed in through the `TrainingArguments` object created above (`trainer_args`), the data, and the metric defined above, we create and pass the following to the `Trainer`:\n",
        "\n",
        "* [data collator](https://huggingface.co/docs/transformers/main_classes/data_collator): groups input into batches\n",
        "* [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback): stops training when performance stops improving"
      ],
      "metadata": {
        "id": "S7kbz8uU-zpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Instantiate the model  \n",
        "mlp = MLP(mlp_config)\n",
        "\n",
        "data_collator = transformers.DataCollatorWithPadding(tokenizer)\n",
        "\n",
        "# Argument gives the number of steps of patience before early stopping\n",
        "early_stopping = transformers.EarlyStoppingCallback(3)\n",
        "\n",
        "trainer = transformers.Trainer(\n",
        "    model=mlp,\n",
        "    args=trainer_args,\n",
        "    train_dataset=dataset[\"train\"],\n",
        "    eval_dataset=dataset[\"validation\"],\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=data_collator,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "AoEoWsj4P_zN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "884b17af-9f07-4cfd-a3ab-c631f11410f8"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "max_steps is given, it will override any value given in num_train_epochs\n",
            "The following columns in the training set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 1864\n",
            "  Num Epochs = 22\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 5000\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='800' max='5000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 800/5000 00:06 < 00:33, 123.96 it/s, Epoch 3/22]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>100</td>\n",
              "      <td>1.148800</td>\n",
              "      <td>1.041087</td>\n",
              "      <td>0.437768</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>200</td>\n",
              "      <td>1.013000</td>\n",
              "      <td>0.986908</td>\n",
              "      <td>0.587983</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>300</td>\n",
              "      <td>0.810100</td>\n",
              "      <td>0.966617</td>\n",
              "      <td>0.515021</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>400</td>\n",
              "      <td>0.672900</td>\n",
              "      <td>0.940674</td>\n",
              "      <td>0.575107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.595600</td>\n",
              "      <td>0.969923</td>\n",
              "      <td>0.583691</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>600</td>\n",
              "      <td>0.407200</td>\n",
              "      <td>1.026756</td>\n",
              "      <td>0.570815</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>700</td>\n",
              "      <td>0.398200</td>\n",
              "      <td>1.052718</td>\n",
              "      <td>0.575107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>800</td>\n",
              "      <td>0.234500</td>\n",
              "      <td>1.097948</td>\n",
              "      <td>0.575107</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to mlp_checkpoints/checkpoint-500\n",
            "Configuration saved in mlp_checkpoints/checkpoint-500/config.json\n",
            "Model weights saved in mlp_checkpoints/checkpoint-500/pytorch_model.bin\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from mlp_checkpoints/checkpoint-500 (score: 0.9699234366416931).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=800, training_loss=0.6600109577178955, metrics={'train_runtime': 6.6371, 'train_samples_per_second': 6026.7, 'train_steps_per_second': 753.338, 'total_flos': 9188898518976.0, 'train_loss': 0.6600109577178955, 'epoch': 3.43})"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can then evaluate the trained model on a given dataset (here our test subset) by calling [`Trainer.evaluate`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.Trainer.evaluate):"
      ],
      "metadata": {
        "id": "Td03fcIa-6Hj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(dataset[\"test\"])\n",
        "\n",
        "pprint(eval_results)"
      ],
      "metadata": {
        "id": "9nlEwpnF2Vow",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        },
        "outputId": "5517e14b-527b-48f1-bc01-4b9bf75fe320"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='30' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [30/30 00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'epoch': 3.43,\n",
            " 'eval_accuracy': 0.6695278969957081,\n",
            " 'eval_loss': 0.8026063442230225,\n",
            " 'eval_runtime': 0.1605,\n",
            " 'eval_samples_per_second': 1451.75,\n",
            " 'eval_steps_per_second': 186.921}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "From the `eval_results` above, we can see that we can specifically grab the accuracy with"
      ],
      "metadata": {
        "id": "ze5NYKfz-8Et"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Accuracy:', eval_results['eval_accuracy'])"
      ],
      "metadata": {
        "id": "8PKRUTav8yGs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "22e0d1cd-5114-431d-f43d-b8e5d445acd5"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.6695278969957081\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save the model for later use\n",
        "\n",
        "1.   `.save_pretrained(path)`\n",
        "2.   `.from_pretrained(path)`\n",
        "\n"
      ],
      "metadata": {
        "id": "P0OYB3TRp-IH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "mlp.save_pretrained(\"mlp-sent\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FosHTDw3p9dd",
        "outputId": "b8c602d1-071a-47f9-c5f9-d7722d78af08"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Configuration saved in mlp-sent/config.json\n",
            "Model weights saved in mlp-sent/pytorch_model.bin\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let us load the model from its saved state and check that it worked well"
      ],
      "metadata": {
        "id": "BRw9MHx-slXS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "mlp2=MLP.from_pretrained(\"mlp-sent\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UKMMNbABqLTg",
        "outputId": "9e48d4f0-d515-4096-ca7b-77935a552c0c"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loading configuration file mlp-sent/config.json\n",
            "Model config MLPConfig {\n",
            "  \"architectures\": [\n",
            "    \"MLP\"\n",
            "  ],\n",
            "  \"hidden_size\": 100,\n",
            "  \"nlabels\": 4,\n",
            "  \"torch_dtype\": \"float32\",\n",
            "  \"transformers_version\": \"4.17.0\",\n",
            "  \"vocab_size\": 50105\n",
            "}\n",
            "\n",
            "loading weights file mlp-sent/pytorch_model.bin\n",
            "All model checkpoint weights were used when initializing MLP.\n",
            "\n",
            "All the weights of MLP were initialized from the model checkpoint at mlp-sent.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use MLP for predictions without further training.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer = transformers.Trainer(\n",
        "    model=mlp2,\n",
        "    args=trainer_args,\n",
        "    train_dataset=dataset[\"train\"],\n",
        "    eval_dataset=dataset[\"validation\"],\n",
        "    compute_metrics=compute_accuracy,\n",
        "    data_collator=data_collator,\n",
        "    callbacks=[early_stopping]\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pfLnUENrrCyp",
        "outputId": "10e4216f-adb8-44f1-c5b4-125bec52510d"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "max_steps is given, it will override any value given in num_train_epochs\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "eval_results = trainer.evaluate(dataset[\"test\"])\n",
        "\n",
        "pprint(eval_results)\n",
        "\n",
        "print('Accuracy:', eval_results['eval_accuracy'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        },
        "id": "hpOM2ZwErMGf",
        "outputId": "4184f06f-586e-4ef2-c610-5f05fb726e49"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the evaluation set  don't have a corresponding argument in `MLP.forward` and have been ignored: token_type_ids, text. If token_type_ids, text are not expected by `MLP.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 233\n",
            "  Batch size = 8\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='30' max='30' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [30/30 00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'eval_accuracy': 0.6695278969957081,\n",
            " 'eval_loss': 0.8026063442230225,\n",
            " 'eval_runtime': 0.1936,\n",
            " 'eval_samples_per_second': 1203.571,\n",
            " 'eval_steps_per_second': 154.966}\n",
            "Accuracy: 0.6695278969957081\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extra time left?\n",
        "\n",
        "* Read through the TrainingArguments documentation, try to understand at least some parts of it https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments\n",
        "* Read through Torch tensor operations, try to understand at least some parts of it: https://pytorch.org/docs/stable/tensors.html\n",
        "* Run the model with different random seeds, how much do the results change?\n",
        "* Run the model with different parameters (hidden layer width, learning rate, etc), how much do the results change?\n"
      ],
      "metadata": {
        "id": "8UT5MV1LtSBh"
      }
    }
  ]
}