social_media_text.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "social_media_text",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "aed0631dc37c477e97da471dbedca363": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d3b9c67bdb8f4f02a6249c9509fa86d2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_384bc3e5d7c54381992e581d826e3d9c",
              "IPY_MODEL_d6676cf190294bd59b32e800b6762b4f"
            ]
          }
        },
        "d3b9c67bdb8f4f02a6249c9509fa86d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "384bc3e5d7c54381992e581d826e3d9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_db02f3efe5754c33b46f5dfcb78eff70",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_054a47ecd58446579e1f437de3d607cd"
          }
        },
        "d6676cf190294bd59b32e800b6762b4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b0018a8ba7c14a25a8b68ede0a0663be",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 770kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d22238fe2614d4bbdf2dd4c40b9995a"
          }
        },
        "db02f3efe5754c33b46f5dfcb78eff70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "054a47ecd58446579e1f437de3d607cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b0018a8ba7c14a25a8b68ede0a0663be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d22238fe2614d4bbdf2dd4c40b9995a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b37cecd1a61a4f6da2ba9c8579ad6083": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_65e83c1ad5f746dfb7647dcb10e5efb8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_59d279290fe14c9d9003fba4ef375d09",
              "IPY_MODEL_c3033b2a15a04c9e8debd9876a8dd0d3"
            ]
          }
        },
        "65e83c1ad5f746dfb7647dcb10e5efb8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "59d279290fe14c9d9003fba4ef375d09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a782a14941dd4cfc83daf007b3000803",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 434,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 434,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5dbb2ce880e54ae191e546c2de0667b4"
          }
        },
        "c3033b2a15a04c9e8debd9876a8dd0d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b7f3466587f042379eaccfa72872cebe",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 434/434 [00:01&lt;00:00, 403B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b955379e35cf4d69a58e9be924c5a086"
          }
        },
        "a782a14941dd4cfc83daf007b3000803": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5dbb2ce880e54ae191e546c2de0667b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b7f3466587f042379eaccfa72872cebe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b955379e35cf4d69a58e9be924c5a086": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "926a586e8fb746619eb5cc5723f88611": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d0882884c40a4d4a82d4893be94f03a0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dfdd59a9d87949bd8e311f10ea985ef2",
              "IPY_MODEL_01bd2fb9896045b39602812faf138fb4"
            ]
          }
        },
        "d0882884c40a4d4a82d4893be94f03a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dfdd59a9d87949bd8e311f10ea985ef2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a08b5ad8b69d40fc9d6cfb2d784fba99",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1344997306,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1344997306,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_48820411dffa4c68be9727b1f7798c15"
          }
        },
        "01bd2fb9896045b39602812faf138fb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_41c7b4bd2d37497da24deb40d44c18d0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.34G/1.34G [00:20&lt;00:00, 66.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_558315d971734f1cba7cd4aa77deed6a"
          }
        },
        "a08b5ad8b69d40fc9d6cfb2d784fba99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "48820411dffa4c68be9727b1f7798c15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "41c7b4bd2d37497da24deb40d44c18d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "558315d971734f1cba7cd4aa77deed6a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "lsW1Djy7tgzS"
      },
      "source": [
        "import io\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d-jVu322ttkN"
      },
      "source": [
        "train_path = '/content/drive/MyDrive/Project 3/Social media text/oct27.train'\n",
        "dev_path = '/content/drive/MyDrive/Project 3/Social media text/oct27.dev'\n",
        "test_path = '/content/drive/MyDrive/Project 3/Social media text/oct27.test'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "saCv-NAy0tV-",
        "outputId": "866950b5-3e64-46d2-93a0-ee214f4a510f"
      },
      "source": [
        "with io.open(train_path, 'r') as f:\n",
        "  train_data = f.readlines()\n",
        "with io.open(dev_path, 'r') as f1:\n",
        "  dev_data = f1.readlines()\n",
        "with io.open(test_path, 'r') as f2:\n",
        "  test_data = f2.readlines()\n",
        "print(train_data[0])\n",
        "print(dev_data[0])\n",
        "print(test_data[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I\tO\n",
            "\n",
            "@ciaranyree\t@\n",
            "\n",
            "RT\t~\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AsWf4bFm6Sox",
        "outputId": "a5b02c11-f442-40bc-c2e4-1eda84f1ac3e"
      },
      "source": [
        "train_data[:50]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I\\tO\\n',\n",
              " 'predict\\tV\\n',\n",
              " 'I\\tO\\n',\n",
              " \"won't\\tV\\n\",\n",
              " 'win\\tV\\n',\n",
              " 'a\\tD\\n',\n",
              " 'single\\tA\\n',\n",
              " 'game\\tN\\n',\n",
              " 'I\\tO\\n',\n",
              " 'bet\\tV\\n',\n",
              " 'on\\tP\\n',\n",
              " '.\\t,\\n',\n",
              " 'Got\\tV\\n',\n",
              " 'Cliff\\t^\\n',\n",
              " 'Lee\\t^\\n',\n",
              " 'today\\tN\\n',\n",
              " ',\\t,\\n',\n",
              " 'so\\tP\\n',\n",
              " 'if\\tP\\n',\n",
              " 'he\\tO\\n',\n",
              " 'loses\\tV\\n',\n",
              " 'its\\tL\\n',\n",
              " 'on\\tP\\n',\n",
              " 'me\\tO\\n',\n",
              " 'RT\\t~\\n',\n",
              " '@e_one\\t@\\n',\n",
              " ':\\t~\\n',\n",
              " 'Texas\\t^\\n',\n",
              " '(\\t,\\n',\n",
              " 'cont\\t~\\n',\n",
              " ')\\t,\\n',\n",
              " 'http://tl.gd/6meogh\\tU\\n',\n",
              " '\\n',\n",
              " 'RT\\t~\\n',\n",
              " '@DjBlack_Pearl\\t@\\n',\n",
              " ':\\t~\\n',\n",
              " 'wat\\tO\\n',\n",
              " 'muhfuckaz\\tN\\n',\n",
              " 'wearin\\tV\\n',\n",
              " '4\\tP\\n',\n",
              " 'the\\tD\\n',\n",
              " 'lingerie\\tN\\n',\n",
              " 'party\\tN\\n',\n",
              " '?????\\t,\\n',\n",
              " '\\n',\n",
              " 'Wednesday\\t^\\n',\n",
              " '27th\\tA\\n',\n",
              " 'october\\t^\\n',\n",
              " '2010\\t$\\n',\n",
              " '.\\t,\\n']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lUztoF9A1g6P"
      },
      "source": [
        "def get_data(data):\n",
        "  sentences = []\n",
        "  labels = []\n",
        "  tokens = []\n",
        "  tags = []\n",
        "  for toktag in data:\n",
        "    if len(toktag.split()) == 2:\n",
        "      tokens.append(toktag.split()[0])\n",
        "      tags.append(toktag.split()[1])\n",
        "    if len(toktag.split()) == 1:\n",
        "      tokens.append(\" \")\n",
        "      tags.append(toktag.split()[0])\n",
        "    if not toktag.split():\n",
        "      sentences.append(tokens)\n",
        "      labels.append(tags)\n",
        "      tokens = []\n",
        "      tags = []\n",
        "  return sentences, labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CvpM79c53ynN"
      },
      "source": [
        "train_sent, train_labels = get_data(train_data)\n",
        "dev_sent, dev_labels = get_data(dev_data)\n",
        "test_sent, test_labels = get_data(test_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZkchOXoU3yq4",
        "outputId": "2e022fd3-3bbc-4a24-a759-f4dfe8fface6"
      },
      "source": [
        "print(train_sent[0])\n",
        "print(train_labels[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['I', 'predict', 'I', \"won't\", 'win', 'a', 'single', 'game', 'I', 'bet', 'on', '.', 'Got', 'Cliff', 'Lee', 'today', ',', 'so', 'if', 'he', 'loses', 'its', 'on', 'me', 'RT', '@e_one', ':', 'Texas', '(', 'cont', ')', 'http://tl.gd/6meogh']\n",
            "['O', 'V', 'O', 'V', 'V', 'D', 'A', 'N', 'O', 'V', 'P', ',', 'V', '^', '^', 'N', ',', 'P', 'P', 'O', 'V', 'L', 'P', 'O', '~', '@', '~', '^', ',', '~', ',', 'U']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zKWuKCFLVgkB",
        "outputId": "a8e63f8c-2268-44fc-c9ee-72e783fc7931"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/84/7bc03215279f603125d844bf81c3fb3f2d50fe8e511546eb4897e4be2067/transformers-4.0.0-py3-none-any.whl (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 8.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 17.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 43.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.11.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=c9be15a898c2bde2d15fa6639beb0ab200feae0196014cc116adf6002e292a2d\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eHsklsl1WpUj"
      },
      "source": [
        "import torch\n",
        "torch.cuda.empty_cache()\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "from torchtext import data\n",
        "from torchtext import datasets\n",
        "\n",
        "from transformers import BertTokenizer, BertModel\n",
        "\n",
        "import numpy as np\n",
        "import time\n",
        "import random\n",
        "import functools"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3umEns1fW1pw"
      },
      "source": [
        "# set the random seeds for reproducability\n",
        "SEED = 1234\n",
        "\n",
        "random.seed(SEED)\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "aed0631dc37c477e97da471dbedca363",
            "d3b9c67bdb8f4f02a6249c9509fa86d2",
            "384bc3e5d7c54381992e581d826e3d9c",
            "d6676cf190294bd59b32e800b6762b4f",
            "db02f3efe5754c33b46f5dfcb78eff70",
            "054a47ecd58446579e1f437de3d607cd",
            "b0018a8ba7c14a25a8b68ede0a0663be",
            "1d22238fe2614d4bbdf2dd4c40b9995a"
          ]
        },
        "id": "j-nqSQDjArFJ",
        "outputId": "0b989edf-f734-41d6-8369-abff5edc35d4"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "aed0631dc37c477e97da471dbedca363",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TWkrI8KOAtcR",
        "outputId": "c703b63b-e91f-4065-b0fe-e05661993f0b"
      },
      "source": [
        "init_token = tokenizer.cls_token\n",
        "pad_token = tokenizer.pad_token\n",
        "unk_token = tokenizer.unk_token\n",
        "\n",
        "print(init_token, pad_token, unk_token)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[CLS] [PAD] [UNK]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IsgRVWVeAv5-",
        "outputId": "455c66dc-bdf5-4c60-b287-9789ac1d3d63"
      },
      "source": [
        "init_token_idx = tokenizer.convert_tokens_to_ids(init_token)\n",
        "pad_token_idx = tokenizer.convert_tokens_to_ids(pad_token)\n",
        "unk_token_idx = tokenizer.convert_tokens_to_ids(unk_token)\n",
        "\n",
        "print(init_token_idx, pad_token_idx, unk_token_idx)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "101 0 100\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H8OLtoIoAxzT",
        "outputId": "9c510280-03fa-499a-847d-7bd7141b390f"
      },
      "source": [
        "max_input_length = tokenizer.max_model_input_sizes['bert-large-uncased']\n",
        "\n",
        "print(max_input_length)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "512\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9xbpTjMEAzU1"
      },
      "source": [
        "def cut_and_convert_to_id(tokens, tokenizer, max_input_length):\n",
        "    tokens = tokens[:max_input_length-1]\n",
        "    tokens = tokenizer.convert_tokens_to_ids(tokens)\n",
        "    return tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cXUNAi0KA0s0"
      },
      "source": [
        "def cut_to_max_length(tokens, max_input_length):\n",
        "    tokens = tokens[:max_input_length-1]\n",
        "    return tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oROHO-apA2M3"
      },
      "source": [
        "text_preprocessor = functools.partial(cut_and_convert_to_id,\n",
        "                                      tokenizer = tokenizer,\n",
        "                                      max_input_length = max_input_length)\n",
        "\n",
        "tag_preprocessor = functools.partial(cut_to_max_length,\n",
        "                                     max_input_length = max_input_length)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Em2rorz8A3uW"
      },
      "source": [
        "TEXT = data.Field(use_vocab = False,\n",
        "                  lower = True,\n",
        "                  preprocessing = text_preprocessor,\n",
        "                  init_token = init_token_idx,\n",
        "                  pad_token = pad_token_idx,\n",
        "                  unk_token = unk_token_idx)\n",
        "\n",
        "TWEET_TAGS = data.Field(unk_token = None,\n",
        "                     init_token = '<pad>',\n",
        "                     preprocessing = tag_preprocessor)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xIl31souBIbl"
      },
      "source": [
        "fields = [(None, None), (\"text\", TEXT), (\"label\", TWEET_TAGS)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262
        },
        "id": "PRI5hZYBBUkH",
        "outputId": "67365ca2-2c30-40d5-ad8b-ffe4b5a5d9cb"
      },
      "source": [
        "text_col = [' '.join([x for x in sent]) for sent in train_sent]\n",
        "label_col = [' '.join([x for x in lab]) for lab in train_labels]\n",
        "dict = {'text': text_col, 'label': label_col}\n",
        "df = pd.DataFrame(dict)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I predict I won't win a single game I bet on ....</td>\n",
              "      <td>O V O V V D A N O V P , V ^ ^ N , P P O V L P ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>RT @DjBlack_Pearl : wat muhfuckaz wearin 4 the...</td>\n",
              "      <td>~ @ ~ O N V P D N N ,</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Wednesday 27th october 2010 . 》have a nice day :)</td>\n",
              "      <td>^ A ^ $ , V D A N E</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>RT @ddlovato : @joejonas oh , hey THANKS jerk !</td>\n",
              "      <td>~ @ ~ @ ! , ! N N ,</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>@thecamion I like monkeys , but I still hate C...</td>\n",
              "      <td>@ O V N , &amp; O R V ^ N N ,</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text                                              label\n",
              "0  I predict I won't win a single game I bet on ....  O V O V V D A N O V P , V ^ ^ N , P P O V L P ...\n",
              "1  RT @DjBlack_Pearl : wat muhfuckaz wearin 4 the...                              ~ @ ~ O N V P D N N ,\n",
              "2  Wednesday 27th october 2010 . 》have a nice day :)                                ^ A ^ $ , V D A N E\n",
              "3    RT @ddlovato : @joejonas oh , hey THANKS jerk !                                ~ @ ~ @ ! , ! N N ,\n",
              "4  @thecamion I like monkeys , but I still hate C...                          @ O V N , & O R V ^ N N ,"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "icWokInNB15k"
      },
      "source": [
        "df.to_csv('/content/drive/MyDrive/Project 3/Social media text/train.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        },
        "id": "vGeC7zvlCHbI",
        "outputId": "9b76d365-3543-4c9b-e39a-f590dcec8260"
      },
      "source": [
        "text_col = [' '.join([x for x in sent]) for sent in dev_sent]\n",
        "label_col = [' '.join([x for x in lab]) for lab in dev_labels]\n",
        "dict = {'text': text_col, 'label': label_col}\n",
        "df = pd.DataFrame(dict)\n",
        "df.head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>@ciaranyree it was on football wives , one of ...</td>\n",
              "      <td>@ O V P N N , $ P D N &amp; D N V ^ ^</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>RT @TheRealQuailman : Currently laughing at La...</td>\n",
              "      <td>~ @ ~ R V P ^ N ,</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>RT @HollywoodOompa : Sat November 6 ill be at ...</td>\n",
              "      <td>~ @ ~ ^ ^ $ L V P ^ N N P D N N N P ~ N L D N ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>You don't know my struggle</td>\n",
              "      <td>O V V D N</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Wind 2.0 mph SE . Barometer 29.676 in , Rising...</td>\n",
              "      <td>N $ N N , N $ N , V , N $ ^ , N N $ N , N $</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>@ShiversTheNinja forgive me for blowing up you...</td>\n",
              "      <td>@ V O P V T D ^ N N ,</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>New Question : How CAN you mend a broken heart...</td>\n",
              "      <td>A N , R V O V D A N , R , V , U</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>@justinbieber can u follow me please ??? it wi...</td>\n",
              "      <td>@ V O V O V , O V V D N P O , E E</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>@JoycieeLovesUuu =) Lls</td>\n",
              "      <td>@ E !</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>@carolyncallahan I knew about it last night , ...</td>\n",
              "      <td>@ O V P O A N , &amp; V V V ^ P L R V V P O D N P ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text                                              label\n",
              "0  @ciaranyree it was on football wives , one of ...                  @ O V P N N , $ P D N & D N V ^ ^\n",
              "1  RT @TheRealQuailman : Currently laughing at La...                                  ~ @ ~ R V P ^ N ,\n",
              "2  RT @HollywoodOompa : Sat November 6 ill be at ...  ~ @ ~ ^ ^ $ L V P ^ N N P D N N N P ~ N L D N ...\n",
              "3                         You don't know my struggle                                          O V V D N\n",
              "4  Wind 2.0 mph SE . Barometer 29.676 in , Rising...        N $ N N , N $ N , V , N $ ^ , N N $ N , N $\n",
              "5  @ShiversTheNinja forgive me for blowing up you...                              @ V O P V T D ^ N N ,\n",
              "6  New Question : How CAN you mend a broken heart...                    A N , R V O V D A N , R , V , U\n",
              "7  @justinbieber can u follow me please ??? it wi...                  @ V O V O V , O V V D N P O , E E\n",
              "8                            @JoycieeLovesUuu =) Lls                                              @ E !\n",
              "9  @carolyncallahan I knew about it last night , ...  @ O V P O A N , & V V V ^ P L R V V P O D N P ..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ViP-uoGJCfIe"
      },
      "source": [
        "df.to_csv('/content/drive/MyDrive/Project 3/Social media text/dev.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 461
        },
        "id": "OQ2wad-ZCpx8",
        "outputId": "c384ff60-36c4-4919-b907-af0f817a513c"
      },
      "source": [
        "text_col = [' '.join([x for x in sent]) for sent in test_sent]\n",
        "label_col = [' '.join([x for x in lab]) for lab in test_labels]\n",
        "dict = {'text': text_col, 'label': label_col}\n",
        "df = pd.DataFrame(dict)\n",
        "df.head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>RT @yomonroe : i want sex_</td>\n",
              "      <td>~ @ ~ O V N</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>My Tummy Feels Funny .. EMILY ! What Are You D...</td>\n",
              "      <td>D N V A , ^ , O V O V P R , E</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Down in the basement with @1cierra22 @lola_sym...</td>\n",
              "      <td>P P D N P @ @ @</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>RT @mtnridge : We are selling Christmas trees ...</td>\n",
              "      <td>~ @ ~ O V V ^ N , V D N U &amp; V T N P D G ~</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I think when I am done with my MFA I should te...</td>\n",
              "      <td>O V R O V V P D N O V V O P N &amp; V D A N P D N ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>@john_mason_ Is there a Flex for CF or Dreamwe...</td>\n",
              "      <td>@ V X D ^ P ^ &amp; ^ P ^ N R , #</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>I just became the mayor of Off 5th Saks Fifth ...</td>\n",
              "      <td>O R V D N P ^ ^ ^ ^ ^ N P @ , U</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>RT @HuffPostCollege : The Lowdown On Four Loko...</td>\n",
              "      <td>~ @ ~ D N P ^ ^ , $ N R P V N N , P P O V O , U</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Some ppl r juz unrealiable .</td>\n",
              "      <td>D N V R A ,</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>@kristenmchugh22 @nerdsinbabeland I'm actually...</td>\n",
              "      <td>@ @ L R D A ^ ^ N , L D A ^ N</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text                                              label\n",
              "0                         RT @yomonroe : i want sex_                                        ~ @ ~ O V N\n",
              "1  My Tummy Feels Funny .. EMILY ! What Are You D...                      D N V A , ^ , O V O V P R , E\n",
              "2  Down in the basement with @1cierra22 @lola_sym...                                    P P D N P @ @ @\n",
              "3  RT @mtnridge : We are selling Christmas trees ...          ~ @ ~ O V V ^ N , V D N U & V T N P D G ~\n",
              "4  I think when I am done with my MFA I should te...  O V R O V V P D N O V V O P N & V D A N P D N ...\n",
              "5  @john_mason_ Is there a Flex for CF or Dreamwe...                      @ V X D ^ P ^ & ^ P ^ N R , #\n",
              "6  I just became the mayor of Off 5th Saks Fifth ...                    O R V D N P ^ ^ ^ ^ ^ N P @ , U\n",
              "7  RT @HuffPostCollege : The Lowdown On Four Loko...    ~ @ ~ D N P ^ ^ , $ N R P V N N , P P O V O , U\n",
              "8                       Some ppl r juz unrealiable .                                        D N V R A ,\n",
              "9  @kristenmchugh22 @nerdsinbabeland I'm actually...                      @ @ L R D A ^ ^ N , L D A ^ N"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yHva2hOvDGQJ"
      },
      "source": [
        "df.to_csv('/content/drive/MyDrive/Project 3/Social media text/test.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eal-FcCJDOwc"
      },
      "source": [
        "from torchtext.data import TabularDataset\n",
        "train_data, dev_data, test_data = TabularDataset.splits(\n",
        "    path='/content/drive/MyDrive/Project 3/Social media text',\n",
        "    train='train.csv', validation='dev.csv', test='test.csv',\n",
        "    format='csv',\n",
        "    skip_header=True,\n",
        "    fields=fields\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w88RYxZ8EFMr",
        "outputId": "ecd01737-ac3e-4d2d-8572-e07744edadfa"
      },
      "source": [
        "print(vars(train_data.examples[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'text': [1045, 16014, 1045, 100, 2663, 1037, 2309, 2208, 1045, 6655, 2006, 1012, 2288, 7656, 3389, 2651, 1010, 2061, 2065, 2002, 12386, 2049, 2006, 2033, 19387, 100, 1024, 3146, 1006, 100, 1007, 100], 'label': ['O', 'V', 'O', 'V', 'V', 'D', 'A', 'N', 'O', 'V', 'P', ',', 'V', '^', '^', 'N', ',', 'P', 'P', 'O', 'V', 'L', 'P', 'O', '~', '@', '~', '^', ',', '~', ',', 'U']}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ETtypCVyEJQy",
        "outputId": "fc669e7d-bd0b-4ca0-8ddf-bf5d4c9fe57b"
      },
      "source": [
        "TWEET_TAGS.build_vocab(train_data)\n",
        "print(TWEET_TAGS.vocab.stoi)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "defaultdict(<function _default_unk_index at 0x7fa3cb3f3620>, {'<pad>': 0, 'V': 1, 'N': 2, ',': 3, 'P': 4, 'O': 5, '^': 6, 'D': 7, 'A': 8, '@': 9, 'R': 10, '~': 11, '!': 12, 'L': 13, '&': 14, 'U': 15, '$': 16, 'E': 17, '#': 18, 'G': 19, 'T': 20, 'Z': 21, 'S': 22, 'X': 23, 'M': 24, 'Y': 25})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6C-Hf9dWEWuT"
      },
      "source": [
        "BATCH_SIZE = 32\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
        "    (train_data, dev_data, test_data),\n",
        "    sort=False,\n",
        "    batch_size=BATCH_SIZE,\n",
        "    device = device\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6oBKfmh5FMOz",
        "outputId": "70be9c32-0b31-416c-b5a5-8ff0f32a681f"
      },
      "source": [
        "print(device)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2hLqcoupFOOK"
      },
      "source": [
        "class BERTPoSTagger(nn.Module):\n",
        "    def __init__(self,\n",
        "                 bert,\n",
        "                 output_dim, \n",
        "                 dropout):\n",
        "        \n",
        "        super().__init__()\n",
        "        \n",
        "        self.bert = bert\n",
        "        \n",
        "        embedding_dim = bert.config.to_dict()['hidden_size']\n",
        "        \n",
        "        self.fc = nn.Linear(embedding_dim, output_dim)\n",
        "        \n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "    def forward(self, text):\n",
        "  \n",
        "        #text = [sent len, batch size]\n",
        "    \n",
        "        text = text.permute(1, 0)\n",
        "        \n",
        "        #text = [batch size, sent len]\n",
        "        \n",
        "        embedded = self.dropout(self.bert(text)[0])\n",
        "        \n",
        "        #embedded = [batch size, seq len, emb dim]\n",
        "                \n",
        "        embedded = embedded.permute(1, 0, 2)\n",
        "                    \n",
        "        #embedded = [sent len, batch size, emb dim]\n",
        "        \n",
        "        predictions = self.fc(self.dropout(embedded))\n",
        "        \n",
        "        #predictions = [sent len, batch size, output dim]\n",
        "        \n",
        "        return predictions"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 114,
          "referenced_widgets": [
            "b37cecd1a61a4f6da2ba9c8579ad6083",
            "65e83c1ad5f746dfb7647dcb10e5efb8",
            "59d279290fe14c9d9003fba4ef375d09",
            "c3033b2a15a04c9e8debd9876a8dd0d3",
            "a782a14941dd4cfc83daf007b3000803",
            "5dbb2ce880e54ae191e546c2de0667b4",
            "b7f3466587f042379eaccfa72872cebe",
            "b955379e35cf4d69a58e9be924c5a086",
            "926a586e8fb746619eb5cc5723f88611",
            "d0882884c40a4d4a82d4893be94f03a0",
            "dfdd59a9d87949bd8e311f10ea985ef2",
            "01bd2fb9896045b39602812faf138fb4",
            "a08b5ad8b69d40fc9d6cfb2d784fba99",
            "48820411dffa4c68be9727b1f7798c15",
            "41c7b4bd2d37497da24deb40d44c18d0",
            "558315d971734f1cba7cd4aa77deed6a"
          ]
        },
        "id": "TlLX_xu9FSV-",
        "outputId": "34f0b4ec-60a9-4711-bc7e-14b48171d036"
      },
      "source": [
        "bert = BertModel.from_pretrained('bert-large-uncased')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b37cecd1a61a4f6da2ba9c8579ad6083",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=434.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "926a586e8fb746619eb5cc5723f88611",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1344997306.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TMpdKHFlFUZh"
      },
      "source": [
        "OUTPUT_DIM = len(TWEET_TAGS.vocab)\n",
        "DROPOUT = 0.25\n",
        "\n",
        "model = BERTPoSTagger(bert,\n",
        "                      OUTPUT_DIM, \n",
        "                      DROPOUT)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Swv-0pEkFeD0",
        "outputId": "8eeec313-cd47-4d28-8fe9-af06c7f29dd2"
      },
      "source": [
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The model has 335,168,538 trainable parameters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ERTDrqlBFgoj"
      },
      "source": [
        "LEARNING_RATE = 3e-5\n",
        "\n",
        "optimizer = optim.Adam(model.parameters(), lr = LEARNING_RATE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FlfVqXX2FlJs"
      },
      "source": [
        "TAG_PAD_IDX = TWEET_TAGS.vocab.stoi[TWEET_TAGS.pad_token]\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=TAG_PAD_IDX)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tsyRRfeQF022"
      },
      "source": [
        "model = model.to(device)\n",
        "criterion = criterion.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "blYlbWNZF_gx"
      },
      "source": [
        "def categorical_accuracy(preds, y, tag_pad_idx):\n",
        "    \n",
        "    max_preds = preds.argmax(dim = 1, keepdim = True) \n",
        "    non_pad_elements = (y != tag_pad_idx).nonzero()\n",
        "    correct = max_preds[non_pad_elements].squeeze(1).eq(y[non_pad_elements])\n",
        "    return correct.sum() / torch.FloatTensor([y[non_pad_elements].shape[0]]).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FjCxA2J6GEMz"
      },
      "source": [
        "def train(model, iterator, optimizer, criterion, tag_pad_idx):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    for batch in iterator:\n",
        "        \n",
        "        text = batch.text\n",
        "        tags = batch.label\n",
        "                \n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        #text = [sent len, batch size]\n",
        "        \n",
        "        predictions = model(text)\n",
        "        \n",
        "        #predictions = [sent len, batch size, output dim]\n",
        "        #tags = [sent len, batch size]\n",
        "        \n",
        "        predictions = predictions.view(-1, predictions.shape[-1])\n",
        "        tags = tags.view(-1)\n",
        "        \n",
        "        #predictions = [sent len * batch size, output dim]\n",
        "        #tags = [sent len * batch size]\n",
        "        \n",
        "        loss = criterion(predictions, tags)\n",
        "                \n",
        "        acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n",
        "        \n",
        "        loss.backward()\n",
        "        \n",
        "        optimizer.step()\n",
        "        \n",
        "        epoch_loss += loss.item()\n",
        "        epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l6mql26NGHvy"
      },
      "source": [
        "def evaluate(model, iterator, criterion, tag_pad_idx):\n",
        "    \n",
        "    epoch_loss = 0\n",
        "    epoch_acc = 0\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for batch in iterator:\n",
        "\n",
        "            text = batch.text\n",
        "            tags = batch.label\n",
        "            \n",
        "            predictions = model(text)\n",
        "            \n",
        "            predictions = predictions.view(-1, predictions.shape[-1])\n",
        "            tags = tags.view(-1)\n",
        "            \n",
        "            loss = criterion(predictions, tags)\n",
        "            \n",
        "            acc = categorical_accuracy(predictions, tags, tag_pad_idx)\n",
        "\n",
        "            epoch_loss += loss.item()\n",
        "            epoch_acc += acc.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SHmvJzqKGJp6"
      },
      "source": [
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sKR6nZ_FGLXM",
        "outputId": "1705d038-8852-42da-bb90-fcb3ec55590e"
      },
      "source": [
        "N_EPOCHS = 20\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "\n",
        "    start_time = time.time()\n",
        "    \n",
        "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)\n",
        "    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)\n",
        "    \n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), '/content/drive/MyDrive/Project 3/Social media text/tweet_BERT.pt')\n",
        "    \n",
        "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
        "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: This overload of nonzero is deprecated:\n",
            "\tnonzero()\n",
            "Consider using one of the following signatures instead:\n",
            "\tnonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:882.)\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 01 | Epoch Time: 0m 25s\n",
            "\tTrain Loss: 2.673 | Train Acc: 19.78%\n",
            "\t Val. Loss: 2.236 |  Val. Acc: 41.61%\n",
            "Epoch: 02 | Epoch Time: 0m 24s\n",
            "\tTrain Loss: 1.587 | Train Acc: 56.55%\n",
            "\t Val. Loss: 0.951 |  Val. Acc: 73.17%\n",
            "Epoch: 03 | Epoch Time: 0m 25s\n",
            "\tTrain Loss: 0.840 | Train Acc: 76.42%\n",
            "\t Val. Loss: 0.668 |  Val. Acc: 81.69%\n",
            "Epoch: 04 | Epoch Time: 0m 24s\n",
            "\tTrain Loss: 0.610 | Train Acc: 82.76%\n",
            "\t Val. Loss: 0.605 |  Val. Acc: 83.77%\n",
            "Epoch: 05 | Epoch Time: 0m 24s\n",
            "\tTrain Loss: 0.509 | Train Acc: 85.46%\n",
            "\t Val. Loss: 0.574 |  Val. Acc: 84.41%\n",
            "Epoch: 06 | Epoch Time: 0m 25s\n",
            "\tTrain Loss: 0.430 | Train Acc: 87.77%\n",
            "\t Val. Loss: 0.566 |  Val. Acc: 84.91%\n",
            "Epoch: 07 | Epoch Time: 0m 25s\n",
            "\tTrain Loss: 0.370 | Train Acc: 89.47%\n",
            "\t Val. Loss: 0.587 |  Val. Acc: 84.29%\n",
            "Epoch: 08 | Epoch Time: 0m 25s\n",
            "\tTrain Loss: 0.323 | Train Acc: 90.77%\n",
            "\t Val. Loss: 0.570 |  Val. Acc: 85.63%\n",
            "Epoch: 09 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.284 | Train Acc: 91.76%\n",
            "\t Val. Loss: 0.593 |  Val. Acc: 85.26%\n",
            "Epoch: 10 | Epoch Time: 0m 25s\n",
            "\tTrain Loss: 0.251 | Train Acc: 92.93%\n",
            "\t Val. Loss: 0.591 |  Val. Acc: 85.61%\n",
            "Epoch: 11 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.203 | Train Acc: 94.08%\n",
            "\t Val. Loss: 0.641 |  Val. Acc: 85.19%\n",
            "Epoch: 12 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.192 | Train Acc: 94.51%\n",
            "\t Val. Loss: 0.653 |  Val. Acc: 85.26%\n",
            "Epoch: 13 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.161 | Train Acc: 95.31%\n",
            "\t Val. Loss: 0.677 |  Val. Acc: 85.25%\n",
            "Epoch: 14 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.143 | Train Acc: 95.80%\n",
            "\t Val. Loss: 0.689 |  Val. Acc: 85.11%\n",
            "Epoch: 15 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.124 | Train Acc: 96.41%\n",
            "\t Val. Loss: 0.721 |  Val. Acc: 85.10%\n",
            "Epoch: 16 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.102 | Train Acc: 97.11%\n",
            "\t Val. Loss: 0.737 |  Val. Acc: 85.24%\n",
            "Epoch: 17 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.093 | Train Acc: 97.34%\n",
            "\t Val. Loss: 0.777 |  Val. Acc: 84.92%\n",
            "Epoch: 18 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.090 | Train Acc: 97.39%\n",
            "\t Val. Loss: 0.780 |  Val. Acc: 84.94%\n",
            "Epoch: 19 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.074 | Train Acc: 97.92%\n",
            "\t Val. Loss: 0.814 |  Val. Acc: 84.96%\n",
            "Epoch: 20 | Epoch Time: 0m 26s\n",
            "\tTrain Loss: 0.068 | Train Acc: 98.15%\n",
            "\t Val. Loss: 0.821 |  Val. Acc: 84.58%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wPrwDZPIGjzr",
        "outputId": "cbd176b8-247d-494d-a84f-f4958ca801cb"
      },
      "source": [
        "model.load_state_dict(torch.load('/content/drive/MyDrive/Project 3/Social media text/tweet_BERT.pt'))\n",
        "\n",
        "test_loss, test_acc = evaluate(model, test_iterator, criterion, TAG_PAD_IDX)\n",
        "\n",
        "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Loss: 0.565 | Test Acc: 84.13%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PIBQORpYG8Fc"
      },
      "source": [
        "def tag_sentence(model, device, sentence, tokenizer, text_field, tag_field):\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    if isinstance(sentence, str):\n",
        "        tokens = tokenizer.tokenize(sentence)\n",
        "    else:\n",
        "        tokens = sentence\n",
        "    \n",
        "    numericalized_tokens = tokenizer.convert_tokens_to_ids(tokens)\n",
        "    numericalized_tokens = [text_field.init_token] + numericalized_tokens\n",
        "        \n",
        "    unk_idx = text_field.unk_token\n",
        "    \n",
        "    unks = [t for t, n in zip(tokens, numericalized_tokens) if n == unk_idx]\n",
        "    \n",
        "    token_tensor = torch.LongTensor(numericalized_tokens)\n",
        "    \n",
        "    token_tensor = token_tensor.unsqueeze(-1).to(device)\n",
        "         \n",
        "    predictions = model(token_tensor)\n",
        "    \n",
        "    top_predictions = predictions.argmax(-1)\n",
        "    \n",
        "    predicted_tags = [tag_field.vocab.itos[t.item()] for t in top_predictions]\n",
        "    \n",
        "    predicted_tags = predicted_tags[1:]\n",
        "        \n",
        "    assert len(tokens) == len(predicted_tags)\n",
        "    \n",
        "    return tokens, predicted_tags, unks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0QWu_58GHU1D",
        "outputId": "1de93f9a-4435-4171-ccca-e7d990ab1812"
      },
      "source": [
        "sentence2 = 'My heart is always breaking for the ghosts that haunt this room.'\n",
        "\n",
        "tokens, tags, unks = tag_sentence(model, \n",
        "                                  device, \n",
        "                                  sentence2,\n",
        "                                  tokenizer,\n",
        "                                  TEXT, \n",
        "                                  TWEET_TAGS)\n",
        "\n",
        "print(unks)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3iBJRB55Hb22",
        "outputId": "4cd4fe84-1fe0-4615-9deb-59fc0256181a"
      },
      "source": [
        "print(\"Pred. Tag\\tToken\\n\")\n",
        "\n",
        "for token, tag in zip(tokens, tags):\n",
        "    print(f\"{tag}\\t\\t{token}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Pred. Tag\tToken\n",
            "\n",
            "D\t\tmy\n",
            "N\t\theart\n",
            "V\t\tis\n",
            "R\t\talways\n",
            "V\t\tbreaking\n",
            "P\t\tfor\n",
            "D\t\tthe\n",
            "N\t\tghosts\n",
            "P\t\tthat\n",
            "V\t\thaunt\n",
            "D\t\tthis\n",
            "N\t\troom\n",
            ",\t\t.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lHQFrydjHc0k"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}