Generate_Formal_Data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Generate Formal Data.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1XHBpZUiSz6ngK60p5OzEsAOhxzkVVExv",
      "authorship_tag": "ABX9TyNQSLtzQ89hT7TRIh6p4RP1",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/NavePnow/Auto-Hashtag-of-Social-Posts/blob/master/Generate_Formal_Data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YRB3R_SISoXK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import re\n",
        "import sys\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\"\"\"Reference\n",
        "https://www.webopedia.com/quick_ref/Twitter_Dictionary_Guide.asp\n",
        "https://www.socialmediatoday.com/content/top-twitter-abbreviations-you-need-know\n",
        "https://digiphile.info/2009/06/11/top-50-twitter-acronyms-abbreviations-and-initialisms/\n",
        "https://bitrebels.com/social/twitter-dictionary-35-twitter-abbreviations/\n",
        "\"\"\"\n",
        "class TextProcessor:\n",
        "    \"\"\" class TextProcessor is a class dealing with multiple methods for processing data\n",
        "    Args:\n",
        "        in_dir: working directory\n",
        "        dictionary_file: dictionary file includes special Twitter terms\n",
        "        hashtag_file: hashtag file includes all hashtags we need to classify\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, in_dir, dictionary_file, hashtag_file):\n",
        "        self.in_dir = in_dir\n",
        "        self.hashtag = set()\n",
        "        self.dictionary = {}\n",
        "        self.dictionary_file = dictionary_file\n",
        "        self.hashtag_file = hashtag_file\n",
        "\n",
        "    \"\"\" load local dictionary and build index\n",
        "    \"\"\"\n",
        "\n",
        "    def load_dictioanry(self):\n",
        "        print('loading dictionary...')\n",
        "        if (not os.path.exists(self.in_dir)):\n",
        "            print(\"wrong file path!\")\n",
        "            sys.exit(2)\n",
        "        f = open(self.in_dir+\"/\"+self.dictionary_file)\n",
        "\n",
        "        # load dictionary and build index\n",
        "        for line in iter(f):\n",
        "            line = line.split(' ', 1)\n",
        "            if line[0].lower() not in self.dictionary:\n",
        "                self.dictionary[line[0].lower()] = line[1].replace(\n",
        "                    '\\n', '').lower()\n",
        "        print('load dictionary successfully')\n",
        "\n",
        "    \"\"\" load local hashtag and build set\n",
        "    \"\"\"\n",
        "\n",
        "    def load_hashtag(self):\n",
        "        print('loading hashtag...')\n",
        "        if (not os.path.exists(self.in_dir)):\n",
        "            print(\"wrong file path!\")\n",
        "            sys.exit(2)\n",
        "        f = open(self.in_dir+\"/\"+self.hashtag_file)\n",
        "\n",
        "        # load dictionary and build index\n",
        "        for line in iter(f):\n",
        "            if line.lower() not in self.hashtag:\n",
        "                self.hashtag.add(line.lower().replace(\n",
        "                    '\\n', '').replace('#', ''))\n",
        "        print('load hashtag successfully')\n",
        "\n",
        "    \"\"\" Irrelevant hashtag filtering\n",
        "    Args:\n",
        "        text: text to be filtered\n",
        "    \n",
        "    Returns:\n",
        "        ' '.join(rst): filtered hashtag\n",
        "    \"\"\"\n",
        "\n",
        "    def del_hashtag(self, text):\n",
        "        tmp_list = str(text).split(',')\n",
        "        rst = []\n",
        "        for i in range(len(tmp_list)):\n",
        "            if (tmp_list[i].lower().replace(' ', '') in self.hashtag):\n",
        "                rst.append(tmp_list[i].lower().replace(' ', ''))\n",
        "        return ', '.join(rst)\n",
        "\n",
        "    \"\"\" Informal language normalization\n",
        "    Args:\n",
        "        text: text to be normailzed\n",
        "    \n",
        "    Returns:\n",
        "        ' '.join(tmp_list): normalized text\n",
        "    \"\"\"\n",
        "\n",
        "    def informal_norm(self, text):\n",
        "        tmp_list = text.split()\n",
        "        for i in range(len(tmp_list)):\n",
        "            if (tmp_list[i].lower() in self.dictionary):\n",
        "                tmp_list[i] = self.dictionary[tmp_list[i].lower()]\n",
        "        return ' '.join(tmp_list)\n",
        "\n",
        "    \"\"\" Irrelevant text tokens filtering\n",
        "    Args:\n",
        "        text: text to be filtered\n",
        "    \n",
        "    Returns:\n",
        "        text: filtered text\n",
        "    \"\"\"\n",
        "\n",
        "    def cleanup(self, text):\n",
        "        # drop http[s]://*\n",
        "        text = re.sub(\n",
        "            r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '', str(text))\n",
        "\n",
        "        # drop something liek @EP_President\n",
        "        text = re.sub(u\"\\@.*?\\s\", '', str(text))\n",
        "\n",
        "        # drop # of hashtag within sentence\n",
        "        text = text.replace('#', ' ')\n",
        "\n",
        "        #  remove emojis\n",
        "        text = text.encode('ascii', 'ignore').decode('ascii')\n",
        "        \n",
        "        return text\n",
        "\n",
        "    \"\"\" drop tweets whose length <= 3\n",
        "    Args:\n",
        "        text: text to be filtered\n",
        "    \n",
        "    Returns:\n",
        "        text: filtered text\n",
        "    \"\"\"\n",
        "\n",
        "    def drop_tweet(self, text):\n",
        "        if (len(str(text).split()) <= 3):\n",
        "            return ''\n",
        "        else:\n",
        "            return text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KfpBgo4w7GWx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class TextClassifier:\n",
        "    \"\"\" class TextClassifier is a class dealing with text classification\n",
        "    Args:\n",
        "        in_dir: working directory\n",
        "        dictionary_file: dictionary file includes special Twitter terms\n",
        "        hashtag_file: hashtag file includes all hashtags we need to classify\n",
        "        input_file: input file is csv format containing crawl data\n",
        "        model_dir: directory of model output\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, in_dir, dictionary_file, hashtag_file, input_file, model_dir=\"model_out\"):\n",
        "        self.in_dir = in_dir\n",
        "        self.hashtag = set()\n",
        "        self.dictionary = {}\n",
        "        self.dictionary_file = dictionary_file\n",
        "        self.hashtag_file = hashtag_file\n",
        "        self.input_file = input_file\n",
        "        self.train_file = None\n",
        "        self.valid_file = None\n",
        "        self.model_dir = model_dir\n",
        "    \n",
        "    \"\"\" load and process the raw crawl data\n",
        "    Returns:\n",
        "        dat.drop(columns=['hashtag']): formal format of data\n",
        "    \"\"\"\n",
        "\n",
        "    def load_raw_data(self):\n",
        "        textprocessor = TextProcessor(\n",
        "            self.in_dir, self.dictionary_file, self.hashtag_file)\n",
        "        textprocessor.load_dictioanry()\n",
        "        textprocessor.load_hashtag()\n",
        "        dat = pd.read_csv(\n",
        "            self.in_dir + '/' + self.input_file, header=None)\n",
        "\n",
        "        dat.columns = ['tweet', 'hashtag']\n",
        "        # n = len(dat)\n",
        "        # nlist = range(0,n)\n",
        "        dat['id'] = None\n",
        "        dat = dat[['id', 'tweet', 'hashtag']]\n",
        "\n",
        "        total = ['id', 'tweet', 'hashtag']\n",
        "        total = total + list(textprocessor.hashtag)\n",
        "        dat = dat.reindex(columns=list(total), fill_value=0)\n",
        "        dat['tweet'] = dat['tweet'].apply(textprocessor.cleanup)\n",
        "        dat['tweet'] = dat['tweet'].apply(textprocessor.informal_norm)\n",
        "\n",
        "        dat['hashtag'] = dat['hashtag'].apply(textprocessor.del_hashtag)\n",
        "        dat = dat.drop(dat[dat['hashtag'].map(len) <\n",
        "                           1].index).reset_index(drop=True)\n",
        "\n",
        "        dat['tweet'] = dat['tweet'].apply(textprocessor.drop_tweet)\n",
        "        dat = dat.drop(dat[dat['tweet'].map(len) <\n",
        "                           1].index).reset_index(drop=True)\n",
        "        n = len(dat)\n",
        "        nlist = range(0, n)\n",
        "        dat['id'] = nlist\n",
        "\n",
        "        # assign label\n",
        "        for i in range(len(dat['hashtag'])):\n",
        "            tmp_list = dat['hashtag'][i].split(\",\")\n",
        "            for j in range(len(tmp_list)):\n",
        "                tmp_list[j] = tmp_list[j].replace(' ', '')\n",
        "                dat[tmp_list[j]][i] = 1\n",
        "        return dat.drop(columns=['hashtag'])\n",
        "\n",
        "    \"\"\" split the data into train and valid data\n",
        "    Args:\n",
        "        data: formal processed crawl data\n",
        "    \"\"\"\n",
        "\n",
        "    def split_data(self,data):\n",
        "        self.train_file, self.valid_file = np.split(\n",
        "            data.sample(frac=1), [int(.7*len(data))])\n",
        "    \n",
        "    \"\"\" save the train and valid data into file\n",
        "    \"\"\"\n",
        "        \n",
        "    def save_to_file(self):\n",
        "        self.train_file.to_csv(self.in_dir + '/' +\n",
        "                               \"train.csv\", index=False, header=True)\n",
        "        self.valid_file.to_csv(self.in_dir + '/' +\n",
        "                               \"valid.csv\", index=False, header=True)\n",
        "\n",
        "    \"\"\" make prediction given text\n",
        "    Args:\n",
        "        text: free text to be classified\n",
        "    \n",
        "    Returns:\n",
        "        rst_list: a list contains top 7 hashtags accroding to the classification\n",
        "    \"\"\"\n",
        "    def predict(self, text):\n",
        "        predictor = BertClassificationPredictor(\n",
        "            model_path=self.in_dir + '/' +self.model_dir,\n",
        "            label_path=self.in_dir + '/labels',  # location for labels.csv file\n",
        "            multi_label=True,\n",
        "            # model_type='xlnet',\n",
        "            do_lower_case=True)\n",
        "        prediction = predictor.predict(str(text))[:7]\n",
        "        rst_list = []\n",
        "        for i in range(len(prediction)):\n",
        "            rst_list.append(prediction[i][0])\n",
        "        return rst_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CId_vF7bd3ob",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9227a52f-a748-4759-e999-f6eaca1b5a6b"
      },
      "source": [
        "!git clone https://github.com/NavePnow/multi-label-classification.git"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'multi-label-classification' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zy127zvJTT80",
        "colab_type": "code",
        "outputId": "bd6a431e-57f8-475c-9845-8080b73138e4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "textclassifier = TextClassifier(\n",
        "    '/content/multi-label-classification', 'dictionary.txt', 'hashtag.txt', 'input.train.text.csv')\n",
        "data = textclassifier.load_raw_data()\n",
        "data\n",
        "textclassifier.split_data(data)\n",
        "textclassifier.save_to_file()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading dictionary...\n",
            "load dictionary successfully\n",
            "loading hashtag...\n",
            "load hashtag successfully\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:63: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}