train_chatbot.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "train_chatbot.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1e4769c47a174dbb80aa31b73d60d6df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_aee2f1e955284382a340e853f05ee468",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f86c9687a0554592996b7c44f77cd447",
              "IPY_MODEL_7076177db590438c9852588e60bc4b7e",
              "IPY_MODEL_796acd6beaa1474eab01d5e4850802b8"
            ]
          }
        },
        "aee2f1e955284382a340e853f05ee468": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f86c9687a0554592996b7c44f77cd447": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0346079543e54c96a5b21149fd739926",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_86f50e6cdaff4eb885681ba1fa22b888"
          }
        },
        "7076177db590438c9852588e60bc4b7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e4379d416f1c4a6dab6edf74b02cb1c4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1042301,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1042301,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aab1c2886e2f47c6ad3f1fb51487dc92"
          }
        },
        "796acd6beaa1474eab01d5e4850802b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_437238a1e32340b5839efbe94a33b5e9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 2.11MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7bd08e7111ed49488564ee8a8d160520"
          }
        },
        "0346079543e54c96a5b21149fd739926": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "86f50e6cdaff4eb885681ba1fa22b888": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e4379d416f1c4a6dab6edf74b02cb1c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aab1c2886e2f47c6ad3f1fb51487dc92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "437238a1e32340b5839efbe94a33b5e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7bd08e7111ed49488564ee8a8d160520": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7afee4e6d42f41238916e61f35321e1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a06b8e6dd8b14d87b1912165339f90db",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cebe23073434472f8b5b1db624b3305b",
              "IPY_MODEL_2935d28eaca1478488aaa271c25bbdeb",
              "IPY_MODEL_6d56763b1d23470c80509da3a85284ae"
            ]
          }
        },
        "a06b8e6dd8b14d87b1912165339f90db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cebe23073434472f8b5b1db624b3305b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_691e1051e3ee4c1fadf9f7ae20c89f98",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_88aaed0fbaf249d398a014b2d8d6c2d8"
          }
        },
        "2935d28eaca1478488aaa271c25bbdeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_af7a458630464148b2207118266b9f33",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c9ea78649d54f19b167d494c8e4c8f2"
          }
        },
        "6d56763b1d23470c80509da3a85284ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7d2f105dbe1b469e9b8f73ac168dcb63",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:00&lt;00:00, 593kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f69b4b074f254e88aa46de31ce174278"
          }
        },
        "691e1051e3ee4c1fadf9f7ae20c89f98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "88aaed0fbaf249d398a014b2d8d6c2d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af7a458630464148b2207118266b9f33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c9ea78649d54f19b167d494c8e4c8f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d2f105dbe1b469e9b8f73ac168dcb63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f69b4b074f254e88aa46de31ce174278": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8121f206a4c44c3aa1459fd19e7dd3db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a05b72b4e53843d1916092f9e709db24",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_56ee63b4f30445d0a6a3543073232bf8",
              "IPY_MODEL_ee199dfabc474c829d637be2677e7bef",
              "IPY_MODEL_3eecd0020ba644dd9932826678f0f340"
            ]
          }
        },
        "a05b72b4e53843d1916092f9e709db24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56ee63b4f30445d0a6a3543073232bf8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fe59676a0ddf448f8caac7ef2137b790",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9c1a76278834c3ba5eda31245f5013a"
          }
        },
        "ee199dfabc474c829d637be2677e7bef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_18b9b5a8a00f4f2baca2a7c99d0bac5c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 665,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 665,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_506d35d3e50c40be831fb61a320048b7"
          }
        },
        "3eecd0020ba644dd9932826678f0f340": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_66d6a8a4d46a43169a663b925de097d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 665/665 [00:00&lt;00:00, 27.2kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2a23c2ddb81b4d649c64d04a44662288"
          }
        },
        "fe59676a0ddf448f8caac7ef2137b790": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9c1a76278834c3ba5eda31245f5013a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "18b9b5a8a00f4f2baca2a7c99d0bac5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "506d35d3e50c40be831fb61a320048b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "66d6a8a4d46a43169a663b925de097d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2a23c2ddb81b4d649c64d04a44662288": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1850e775a44548d195d66fc009cd07fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6406dc52342b433aaa78e20149b0763b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_aed25b2f8dd545c79fd0a5a90b5949c7",
              "IPY_MODEL_aed2f4cca152466aa0d046f8b606daf4",
              "IPY_MODEL_5aa540e6f4304cdc88414b7a3b64326a"
            ]
          }
        },
        "6406dc52342b433aaa78e20149b0763b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aed25b2f8dd545c79fd0a5a90b5949c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_77e9ab36dbe9441ebb3e20cee0d10bf7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_554126eb81614275b11850ec40c7dc95"
          }
        },
        "aed2f4cca152466aa0d046f8b606daf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ebdc73b27d7d468bb4ae38477dd09f98",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 548118077,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 548118077,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_feb70f9c99644152bd091034512723e7"
          }
        },
        "5aa540e6f4304cdc88414b7a3b64326a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_77bc3b7af1d5457996be0d1b03cdc5c3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 548M/548M [00:09&lt;00:00, 60.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6c3b981339264f65913bf6c3d93c6d55"
          }
        },
        "77e9ab36dbe9441ebb3e20cee0d10bf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "554126eb81614275b11850ec40c7dc95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ebdc73b27d7d468bb4ae38477dd09f98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "feb70f9c99644152bd091034512723e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "77bc3b7af1d5457996be0d1b03cdc5c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6c3b981339264f65913bf6c3d93c6d55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4V30dDc6wn07",
        "outputId": "1535b52b-23f8-4392-b28b-31991acb6025"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mon Feb 14 17:24:33 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   33C    P0    24W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "il-IQ1A4UJgP",
        "outputId": "4272ac1c-f7ec-4b98-eeea-264522bcc9d7"
      },
      "source": [
        "!pip install transformers'==4.2.2'"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers==4.2.2\n",
            "  Downloading transformers-4.2.2-py3-none-any.whl (1.8 MB)\n",
            "\u001b[?25l\r\u001b[K     |▏                               | 10 kB 30.6 MB/s eta 0:00:01\r\u001b[K     |▍                               | 20 kB 34.7 MB/s eta 0:00:01\r\u001b[K     |▋                               | 30 kB 28.0 MB/s eta 0:00:01\r\u001b[K     |▊                               | 40 kB 21.5 MB/s eta 0:00:01\r\u001b[K     |█                               | 51 kB 17.0 MB/s eta 0:00:01\r\u001b[K     |█▏                              | 61 kB 15.4 MB/s eta 0:00:01\r\u001b[K     |█▎                              | 71 kB 14.6 MB/s eta 0:00:01\r\u001b[K     |█▌                              | 81 kB 16.1 MB/s eta 0:00:01\r\u001b[K     |█▊                              | 92 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█▉                              | 102 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██                              | 112 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██▎                             | 122 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██▍                             | 133 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██▋                             | 143 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██▉                             | 153 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███                             | 163 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███▏                            | 174 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███▍                            | 184 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███▌                            | 194 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███▊                            | 204 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████                            | 215 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████                            | 225 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████▎                           | 235 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████▌                           | 245 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████▋                           | 256 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████▉                           | 266 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████                           | 276 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 286 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 296 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 307 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 317 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████                          | 327 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 337 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 348 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 358 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 368 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 378 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████                         | 389 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 399 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 409 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 419 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 430 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████                        | 440 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 450 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 460 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 471 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 481 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 491 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 501 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 512 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 522 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 532 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 542 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████                      | 552 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 563 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 573 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 583 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 593 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████                     | 604 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 614 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 624 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 634 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 645 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 655 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 665 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 675 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 686 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 696 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 706 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 716 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 727 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 737 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 747 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 757 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 768 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 778 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 788 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 798 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 808 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 819 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 829 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 839 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 849 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 860 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 870 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 880 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 890 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 901 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 911 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 921 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 931 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 942 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 952 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 962 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 972 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 983 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 993 kB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 1.0 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 1.0 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 1.0 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 1.0 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.0 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 1.1 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 1.2 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 1.3 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 1.4 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.5 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 1.6 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 1.7 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 1.8 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.8 MB 14.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.8 MB 14.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.2) (2.23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.2) (1.19.5)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.2) (4.10.1)\n",
            "Collecting tokenizers==0.9.4\n",
            "  Downloading tokenizers-0.9.4-cp37-cp37m-manylinux2010_x86_64.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 72.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.2) (21.3)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.47-py2.py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 67.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.2) (4.62.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.2) (3.4.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.2) (2019.12.20)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.2.2) (3.7.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.2.2) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.2.2) (3.0.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.2.2) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.2.2) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.2.2) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.2.2) (1.24.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.2.2) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.2.2) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.2.2) (1.1.0)\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.47 tokenizers-0.9.4 transformers-4.2.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fFHWKDEoIuk"
      },
      "source": [
        "import torch\n",
        "from transformers import (\n",
        "    GPT2Tokenizer, GPT2LMHeadModel, TextDataset, LineByLineTextDataset,\n",
        "    DataCollatorForLanguageModeling, TrainingArguments, Trainer, get_cosine_schedule_with_warmup\n",
        ")\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import json\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import json\n",
        "import nltk\n"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hh6fjAqjpkrq",
        "outputId": "8728dbd0-8248-48c0-f261-7a6108bb2cbb"
      },
      "source": [
        "!wget https://s3.amazonaws.com/datasets.huggingface.co/personachat/personachat_self_original.json"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-02-14 17:24:49--  https://s3.amazonaws.com/datasets.huggingface.co/personachat/personachat_self_original.json\n",
            "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.217.10.86\n",
            "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.217.10.86|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 209850483 (200M) [application/json]\n",
            "Saving to: ‘personachat_self_original.json’\n",
            "\n",
            "personachat_self_or 100%[===================>] 200.13M  34.9MB/s    in 6.4s    \n",
            "\n",
            "2022-02-14 17:24:56 (31.4 MB/s) - ‘personachat_self_original.json’ saved [209850483/209850483]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xg0HHhFFRX6k"
      },
      "source": [
        "bos_token = '<start>'\n",
        "eos_token = '<end>'"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YD4qBeqOp51m"
      },
      "source": [
        "with open('/content/personachat_self_original.json') as json_file:\n",
        "    data = json.load(json_file)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chats = []\n",
        "for i in range(len(data['train'])):\n",
        "    personality = \" \".join(data['train'][i]['personality'])\n",
        "    chat = data['train'][i]['utterances'][-1]['history']\n",
        "    final_text = bos_token + personality + '<start_chat>'\n",
        "    for j in range(len(chat)):\n",
        "        if j % 2 == 0:\n",
        "            final_text += '<Human>'\n",
        "        else:\n",
        "            final_text += '<AI>'\n",
        "        final_text += chat[j]\n",
        "    chats.append(final_text + eos_token)"
      ],
      "metadata": {
        "id": "GJQpDSUejPMh"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chats = \" \".join(chats)"
      ],
      "metadata": {
        "id": "jpccGWtrlglm"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data_path = 'chats.txt'\n",
        "with open(train_data_path, 'w') as t:\n",
        "    t.write(chats)"
      ],
      "metadata": {
        "id": "MMr2ug4ylSa3"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y1S4cjsQnLxG"
      },
      "source": [
        "# Tokenezation\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98,
          "referenced_widgets": [
            "1e4769c47a174dbb80aa31b73d60d6df",
            "aee2f1e955284382a340e853f05ee468",
            "f86c9687a0554592996b7c44f77cd447",
            "7076177db590438c9852588e60bc4b7e",
            "796acd6beaa1474eab01d5e4850802b8",
            "0346079543e54c96a5b21149fd739926",
            "86f50e6cdaff4eb885681ba1fa22b888",
            "e4379d416f1c4a6dab6edf74b02cb1c4",
            "aab1c2886e2f47c6ad3f1fb51487dc92",
            "437238a1e32340b5839efbe94a33b5e9",
            "7bd08e7111ed49488564ee8a8d160520",
            "7afee4e6d42f41238916e61f35321e1e",
            "a06b8e6dd8b14d87b1912165339f90db",
            "cebe23073434472f8b5b1db624b3305b",
            "2935d28eaca1478488aaa271c25bbdeb",
            "6d56763b1d23470c80509da3a85284ae",
            "691e1051e3ee4c1fadf9f7ae20c89f98",
            "88aaed0fbaf249d398a014b2d8d6c2d8",
            "af7a458630464148b2207118266b9f33",
            "1c9ea78649d54f19b167d494c8e4c8f2",
            "7d2f105dbe1b469e9b8f73ac168dcb63",
            "f69b4b074f254e88aa46de31ce174278"
          ]
        },
        "id": "mkJPniCBUpqQ",
        "outputId": "049a8a17-88b3-4895-aa07-1621f6648602"
      },
      "source": [
        "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\", bos_token=bos_token, eos_token=eos_token, pad_token='<|pad|>')"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1e4769c47a174dbb80aa31b73d60d6df",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7afee4e6d42f41238916e61f35321e1e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j8FItmRBWSxS"
      },
      "source": [
        "block_size = 1024"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jb4j8jmzvOc1",
        "outputId": "6cb86235-afc8-42f2-a60f-f37e5280b165"
      },
      "source": [
        "train_dataset = TextDataset(tokenizer=tokenizer, file_path='/content/chats.txt', block_size=block_size, overwrite_cache=True)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/data/datasets/language_modeling.py:58: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rjIvTs1DQwf8",
        "outputId": "fc48a97a-2da0-483d-d53e-1c7e060d7565"
      },
      "source": [
        "len(train_dataset)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4283"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_O_n3PzynV_Q"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bf3jNlt6r-WK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "8121f206a4c44c3aa1459fd19e7dd3db",
            "a05b72b4e53843d1916092f9e709db24",
            "56ee63b4f30445d0a6a3543073232bf8",
            "ee199dfabc474c829d637be2677e7bef",
            "3eecd0020ba644dd9932826678f0f340",
            "fe59676a0ddf448f8caac7ef2137b790",
            "d9c1a76278834c3ba5eda31245f5013a",
            "18b9b5a8a00f4f2baca2a7c99d0bac5c",
            "506d35d3e50c40be831fb61a320048b7",
            "66d6a8a4d46a43169a663b925de097d6",
            "2a23c2ddb81b4d649c64d04a44662288",
            "1850e775a44548d195d66fc009cd07fd",
            "6406dc52342b433aaa78e20149b0763b",
            "aed25b2f8dd545c79fd0a5a90b5949c7",
            "aed2f4cca152466aa0d046f8b606daf4",
            "5aa540e6f4304cdc88414b7a3b64326a",
            "77e9ab36dbe9441ebb3e20cee0d10bf7",
            "554126eb81614275b11850ec40c7dc95",
            "ebdc73b27d7d468bb4ae38477dd09f98",
            "feb70f9c99644152bd091034512723e7",
            "77bc3b7af1d5457996be0d1b03cdc5c3",
            "6c3b981339264f65913bf6c3d93c6d55"
          ]
        },
        "outputId": "31ff7433-757e-4392-baf4-aa35dc3d856f"
      },
      "source": [
        "model = GPT2LMHeadModel.from_pretrained(\"gpt2\")"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8121f206a4c44c3aa1459fd19e7dd3db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1850e775a44548d195d66fc009cd07fd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/548M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uHOrlUse4wm9"
      },
      "source": [
        "model.cuda()\n",
        "None"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BZ84KrfeVTLJ",
        "outputId": "a3f0d307-6a81-4c92-a566-834874319427"
      },
      "source": [
        "model.resize_token_embeddings(len(tokenizer))"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Embedding(50260, 768)"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Nou2PgYEKJr"
      },
      "source": [
        "learning_rate = 2e-5\n",
        "batch_size = 1\n",
        "num_epochs = 4"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-zyz1nIHbsrd"
      },
      "source": [
        "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)\n",
        "\n",
        "seed = 567\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='chats_output',\n",
        "    overwrite_output_dir=True,\n",
        "    do_train=True,\n",
        "    num_train_epochs=num_epochs,\n",
        "    per_device_train_batch_size=batch_size,\n",
        "    prediction_loss_only=True,\n",
        "    logging_steps=100,\n",
        "    load_best_model_at_end=True,\n",
        "    #save_steps=300,\n",
        "    seed=seed,\n",
        "    learning_rate=learning_rate\n",
        ")"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "76wLGGclb1cQ"
      },
      "source": [
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    tokenizer=tokenizer,\n",
        "    args=training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=train_dataset\n",
        ")\n",
        "\n",
        "train_dataloader = trainer.get_train_dataloader()\n",
        "num_train_steps = len(train_dataloader) * num_epochs\n",
        "trainer.create_optimizer_and_scheduler(num_train_steps) \n",
        "\n",
        "trainer.lr_scheduler = get_cosine_schedule_with_warmup(\n",
        "    trainer.optimizer,\n",
        "    num_warmup_steps=num_train_steps//10,\n",
        "    num_training_steps=num_train_steps\n",
        ")"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bSdoqrdNDune"
      },
      "source": [
        "Заводим!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Pq5ZPdE1b8c9",
        "outputId": "0a1b003f-edda-45af-d9d8-a525b7685db1"
      },
      "source": [
        "trainer.train() "
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='17132' max='17132' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [17132/17132 35:58, Epoch 4/4]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>100</td>\n",
              "      <td>85.025500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>200</td>\n",
              "      <td>16.056700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>300</td>\n",
              "      <td>4.340300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>400</td>\n",
              "      <td>3.583700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>3.257400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>600</td>\n",
              "      <td>3.096200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>700</td>\n",
              "      <td>2.949600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>800</td>\n",
              "      <td>2.877400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>900</td>\n",
              "      <td>2.799900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>2.747100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1100</td>\n",
              "      <td>2.722200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1200</td>\n",
              "      <td>2.703500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1300</td>\n",
              "      <td>2.669800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1400</td>\n",
              "      <td>2.641900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>2.631400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1600</td>\n",
              "      <td>2.626800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1700</td>\n",
              "      <td>2.594000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1800</td>\n",
              "      <td>2.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1900</td>\n",
              "      <td>2.557400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>2.579200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2100</td>\n",
              "      <td>2.556800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2200</td>\n",
              "      <td>2.530400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2300</td>\n",
              "      <td>2.535500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2400</td>\n",
              "      <td>2.530000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>2.547600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2600</td>\n",
              "      <td>2.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2700</td>\n",
              "      <td>2.495900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2800</td>\n",
              "      <td>2.488400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2900</td>\n",
              "      <td>2.476600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>2.470200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3100</td>\n",
              "      <td>2.467800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3200</td>\n",
              "      <td>2.419100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3300</td>\n",
              "      <td>2.441700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3400</td>\n",
              "      <td>2.427600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>2.420500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3600</td>\n",
              "      <td>2.459300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3700</td>\n",
              "      <td>2.409200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3800</td>\n",
              "      <td>2.409700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3900</td>\n",
              "      <td>2.404200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>2.384700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4100</td>\n",
              "      <td>2.416000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4200</td>\n",
              "      <td>2.390600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4300</td>\n",
              "      <td>2.353700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4400</td>\n",
              "      <td>2.341500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>2.342700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4600</td>\n",
              "      <td>2.318200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4700</td>\n",
              "      <td>2.343500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4800</td>\n",
              "      <td>2.353800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4900</td>\n",
              "      <td>2.344700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>2.339500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5100</td>\n",
              "      <td>2.318300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5200</td>\n",
              "      <td>2.314000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5300</td>\n",
              "      <td>2.317600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5400</td>\n",
              "      <td>2.323700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>2.317400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5600</td>\n",
              "      <td>2.329700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5700</td>\n",
              "      <td>2.332900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5800</td>\n",
              "      <td>2.311100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5900</td>\n",
              "      <td>2.300100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>2.306600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6100</td>\n",
              "      <td>2.275700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6200</td>\n",
              "      <td>2.265600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6300</td>\n",
              "      <td>2.312500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6400</td>\n",
              "      <td>2.280100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>2.257700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6600</td>\n",
              "      <td>2.287800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6700</td>\n",
              "      <td>2.275700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6800</td>\n",
              "      <td>2.270000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6900</td>\n",
              "      <td>2.251000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>2.265700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7100</td>\n",
              "      <td>2.283000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7200</td>\n",
              "      <td>2.264100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7300</td>\n",
              "      <td>2.255400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7400</td>\n",
              "      <td>2.249500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>2.282600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7600</td>\n",
              "      <td>2.247600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7700</td>\n",
              "      <td>2.262600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7800</td>\n",
              "      <td>2.250000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7900</td>\n",
              "      <td>2.254700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>2.249300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8100</td>\n",
              "      <td>2.258100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8200</td>\n",
              "      <td>2.268700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8300</td>\n",
              "      <td>2.216400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8400</td>\n",
              "      <td>2.265900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>2.225700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8600</td>\n",
              "      <td>2.227700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8700</td>\n",
              "      <td>2.200800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8800</td>\n",
              "      <td>2.204300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8900</td>\n",
              "      <td>2.195100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>2.215800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9100</td>\n",
              "      <td>2.189400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9200</td>\n",
              "      <td>2.224700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9300</td>\n",
              "      <td>2.196000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9400</td>\n",
              "      <td>2.204400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>2.203400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9600</td>\n",
              "      <td>2.184000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9700</td>\n",
              "      <td>2.201800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9800</td>\n",
              "      <td>2.167700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9900</td>\n",
              "      <td>2.186200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10000</td>\n",
              "      <td>2.192700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10100</td>\n",
              "      <td>2.176700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10200</td>\n",
              "      <td>2.166600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10300</td>\n",
              "      <td>2.201800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10400</td>\n",
              "      <td>2.191200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10500</td>\n",
              "      <td>2.188800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10600</td>\n",
              "      <td>2.168200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10700</td>\n",
              "      <td>2.179100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10800</td>\n",
              "      <td>2.198900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10900</td>\n",
              "      <td>2.181700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11000</td>\n",
              "      <td>2.192600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11100</td>\n",
              "      <td>2.174700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11200</td>\n",
              "      <td>2.162900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11300</td>\n",
              "      <td>2.169500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11400</td>\n",
              "      <td>2.171400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11500</td>\n",
              "      <td>2.176700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11600</td>\n",
              "      <td>2.165400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11700</td>\n",
              "      <td>2.158400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11800</td>\n",
              "      <td>2.163100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11900</td>\n",
              "      <td>2.181800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12000</td>\n",
              "      <td>2.165600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12100</td>\n",
              "      <td>2.179500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12200</td>\n",
              "      <td>2.174900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12300</td>\n",
              "      <td>2.159000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12400</td>\n",
              "      <td>2.167700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12500</td>\n",
              "      <td>2.168300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12600</td>\n",
              "      <td>2.184100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12700</td>\n",
              "      <td>2.155100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12800</td>\n",
              "      <td>2.156800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12900</td>\n",
              "      <td>2.165900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13000</td>\n",
              "      <td>2.143800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13100</td>\n",
              "      <td>2.159100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13200</td>\n",
              "      <td>2.151300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13300</td>\n",
              "      <td>2.164800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13400</td>\n",
              "      <td>2.160800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13500</td>\n",
              "      <td>2.131000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13600</td>\n",
              "      <td>2.149300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13700</td>\n",
              "      <td>2.177800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13800</td>\n",
              "      <td>2.113700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13900</td>\n",
              "      <td>2.117700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14000</td>\n",
              "      <td>2.139900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14100</td>\n",
              "      <td>2.168900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14200</td>\n",
              "      <td>2.149300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14300</td>\n",
              "      <td>2.149000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14400</td>\n",
              "      <td>2.127300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14500</td>\n",
              "      <td>2.186800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14600</td>\n",
              "      <td>2.148100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14700</td>\n",
              "      <td>2.136200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14800</td>\n",
              "      <td>2.104500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14900</td>\n",
              "      <td>2.109300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15000</td>\n",
              "      <td>2.155500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15100</td>\n",
              "      <td>2.150500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15200</td>\n",
              "      <td>2.128800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15300</td>\n",
              "      <td>2.149800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15400</td>\n",
              "      <td>2.161100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15500</td>\n",
              "      <td>2.128800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15600</td>\n",
              "      <td>2.153900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15700</td>\n",
              "      <td>2.118300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15800</td>\n",
              "      <td>2.114300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15900</td>\n",
              "      <td>2.140000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16000</td>\n",
              "      <td>2.141700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16100</td>\n",
              "      <td>2.146300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16200</td>\n",
              "      <td>2.139900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16300</td>\n",
              "      <td>2.125900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16400</td>\n",
              "      <td>2.129600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16500</td>\n",
              "      <td>2.153900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16600</td>\n",
              "      <td>2.135500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16700</td>\n",
              "      <td>2.148100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16800</td>\n",
              "      <td>2.140100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16900</td>\n",
              "      <td>2.134500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17000</td>\n",
              "      <td>2.144000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17100</td>\n",
              "      <td>2.145300</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=17132, training_loss=2.874273501014754, metrics={'train_runtime': 2159.0167, 'train_samples_per_second': 7.935, 'total_flos': 13098653262544896, 'epoch': 4.0})"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kgT-Z-wGFj8O"
      },
      "source": [
        "# Save model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_gFW1bS5Fr3U",
        "outputId": "d1a658fc-3ca9-4828-d5ec-dec80c8ca5d0"
      },
      "source": [
        "output_dir = '/content/chats_output'\n",
        "\n",
        "model.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('/content/jokes_output/tokenizer_config.json',\n",
              " '/content/jokes_output/special_tokens_map.json',\n",
              " '/content/jokes_output/vocab.json',\n",
              " '/content/jokes_output/merges.txt',\n",
              " '/content/jokes_output/added_tokens.json')"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tYcS6UTnE6Gu"
      },
      "source": [
        "# Text generation\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s8_y37NeJc3u"
      },
      "source": [
        "def generate(prompt):\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    input_ids = tokenizer.encode(prompt, return_tensors='pt')\n",
        "    input_ids = input_ids.to('cuda') \n",
        "    eos_id = tokenizer.encode(eos_token)[0]\n",
        "\n",
        "    sample_outputs = model.generate(\n",
        "        input_ids, \n",
        "        max_length= len(input_ids[0]) + 50, \n",
        "        min_length=len(input_ids[0]) + 30,\n",
        "        do_sample=True,\n",
        "        top_k=50,\n",
        "        top_p=0.92, \n",
        "        temperature=0.95,\n",
        "        num_beams=1,\n",
        "        repition_penalty=3,\n",
        "        eos_token_id=eos_id,\n",
        "        num_return_sequences=1\n",
        "        )\n",
        "    for i, sample_output in enumerate(sample_outputs):\n",
        "          sample_output = sample_output[len(input_ids[0]):]\n",
        "          return tokenizer.decode(sample_output, skip_special_tokens=False)"
      ],
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "JZKW0SiNueZ5"
      },
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = '<start>i am a musician and hope to make it big some day . i play the piano and guitar and sing . i also work as a custodian to help pay the bills . my favorite type of music to sing is folk music .<start_chat>'\n",
        "while True:\n",
        "    prompt += '<Human>'\n",
        "    prompt += input('Input: ') + '<AI>'\n",
        "    generated = generate(prompt)\n",
        "    if '<Human>' in generated:\n",
        "        generated = generated[: generated.find('<Human>')]\n",
        "    print(f'AI: {generated}')\n",
        "    prompt += generated"
      ],
      "metadata": {
        "id": "yPuu-693vFmt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DwF_pKBm88b0",
        "outputId": "8d11aa2d-3f1e-4afb-ad8d-cc5c50528739"
      },
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    }
  ]
}