FastText2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.models import Sequential,load_model\n",
    "from keras.layers import Dense,Embedding\n",
    "from keras.layers import LSTM,Dropout\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import load_model\n",
    "from sklearn.model_selection import train_test_split\n",
    "import fasttext\n",
    "import os\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "import fasttext.util\n",
    "from gensim.models import KeyedVectors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('desks', 0.7923153638839722), ('Desk', 0.6869951486587524), ('desk.', 0.6602819561958313), ('desk-', 0.6187258958816528), ('credenza', 0.5955315828323364), ('roll-top', 0.5875717401504517), ('rolltop', 0.5837830305099487), ('bookshelf', 0.5758029222488403), ('Desks', 0.5755287408828735), ('sofa', 0.5617446899414062)]\n"
     ]
    }
   ],
   "source": [
    "model = KeyedVectors.load_word2vec_format('wiki-news-300d-1M.vec')\n",
    "print (model.most_similar('desk'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_csv('offensiveMemes.csv')\n",
    "data.text_corrected=data.text_corrected.astype(str)\n",
    "data[\"text_corrected\"]= data[\"text_corrected\"].str.lower() \n",
    "\n",
    "data=data.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "\n",
    "data['text_corrected'] = data['text_corrected'].apply(lambda x: re.sub('[^a-zA-z0-9\\s]', '', x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_f=pd.DataFrame(data, columns = ['text_corrected', 'offensive'])\n",
    "\n",
    "data_f = data_f.rename(columns={\"text_corrected\":\"text\", \"offensive\":\"label\"})\n",
    "\n",
    "\n",
    "data_f['label'] = '__label__' + data_f['label'].astype(str)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_f\n",
    "a=data_f[data_f['label']=='__label__not_offensive']\n",
    "b=data_f[data_f['label']=='__label__very_offensive']\n",
    "c=data_f[data_f['label']=='__label__slight']\n",
    "d=data_f[data_f['label']=='__label__hateful_offensive']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "test=a.tail(40)\n",
    "test=test.append(b.tail(40))\n",
    "test=test.append(c.tail(40))\n",
    "test=test.append(d.tail(40))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>my girlfriend wanted to travel to the moon i s...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>whats lantern up to these days its sad when a ...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>even likes are thanos marvelousjokes odd likes...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>cat memes meow  meme machine</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>no new baby godfather memes dies memegeneratornet</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6980</th>\n",
       "      <td>raindrop drop top imgflipcom cspan hitler neve...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6983</th>\n",
       "      <td>if someones having a bad day heres a babydory ...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6988</th>\n",
       "      <td>puberty was doing so good what happened imgfli...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6989</th>\n",
       "      <td>todays family feud name something you are sick...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6990</th>\n",
       "      <td>mein darling jew must be this tallto reicht ma...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2713 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  \\\n",
       "1     my girlfriend wanted to travel to the moon i s...   \n",
       "2     whats lantern up to these days its sad when a ...   \n",
       "4     even likes are thanos marvelousjokes odd likes...   \n",
       "10                         cat memes meow  meme machine   \n",
       "13    no new baby godfather memes dies memegeneratornet   \n",
       "...                                                 ...   \n",
       "6980  raindrop drop top imgflipcom cspan hitler neve...   \n",
       "6983  if someones having a bad day heres a babydory ...   \n",
       "6988  puberty was doing so good what happened imgfli...   \n",
       "6989  todays family feud name something you are sick...   \n",
       "6990  mein darling jew must be this tallto reicht ma...   \n",
       "\n",
       "                       label  \n",
       "1     __label__not_offensive  \n",
       "2     __label__not_offensive  \n",
       "4     __label__not_offensive  \n",
       "10    __label__not_offensive  \n",
       "13    __label__not_offensive  \n",
       "...                      ...  \n",
       "6980  __label__not_offensive  \n",
       "6983  __label__not_offensive  \n",
       "6988  __label__not_offensive  \n",
       "6989  __label__not_offensive  \n",
       "6990  __label__not_offensive  \n",
       "\n",
       "[2713 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=a.head(200)\n",
    "train=train.append(b.head(200))\n",
    "train=train.append(c.head(200))\n",
    "train=train.append(d.head(200))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>my girlfriend wanted to travel to the moon i s...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>whats lantern up to these days its sad when a ...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>even likes are thanos marvelousjokes odd likes...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>cat memes meow  meme machine</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>no new baby godfather memes dies memegeneratornet</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5918</th>\n",
       "      <td>when u sneeze in front of ur pet and they look...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5937</th>\n",
       "      <td>the modi government has decided to include 15 ...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6010</th>\n",
       "      <td>world leader 10year challenge 2009 2019 even w...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6068</th>\n",
       "      <td>theres a dog behind you what does he look like</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6070</th>\n",
       "      <td>god creates jews god creates jesus jews kills ...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>800 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  \\\n",
       "1     my girlfriend wanted to travel to the moon i s...   \n",
       "2     whats lantern up to these days its sad when a ...   \n",
       "4     even likes are thanos marvelousjokes odd likes...   \n",
       "10                         cat memes meow  meme machine   \n",
       "13    no new baby godfather memes dies memegeneratornet   \n",
       "...                                                 ...   \n",
       "5918  when u sneeze in front of ur pet and they look...   \n",
       "5937  the modi government has decided to include 15 ...   \n",
       "6010  world leader 10year challenge 2009 2019 even w...   \n",
       "6068     theres a dog behind you what does he look like   \n",
       "6070  god creates jews god creates jesus jews kills ...   \n",
       "\n",
       "                           label  \n",
       "1         __label__not_offensive  \n",
       "2         __label__not_offensive  \n",
       "4         __label__not_offensive  \n",
       "10        __label__not_offensive  \n",
       "13        __label__not_offensive  \n",
       "...                          ...  \n",
       "5918  __label__hateful_offensive  \n",
       "5937  __label__hateful_offensive  \n",
       "6010  __label__hateful_offensive  \n",
       "6068  __label__hateful_offensive  \n",
       "6070  __label__hateful_offensive  \n",
       "\n",
       "[800 rows x 2 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv('train1.txt', sep='\\t', index = False, header = False)\n",
    "test.to_csv('test1.txt', sep='\\t', index = False, header = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data_path=os.getcwd()+'/train1.txt'\n",
    "testing_data_path=os.getcwd()+'/test1.txt'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/askvyas/Documents/Memology/train1.txt'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'_FastText' object has no attribute 'train_supervised'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-83-883a498da9b3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_supervised\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_data_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mhyper_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: '_FastText' object has no attribute 'train_supervised'"
     ]
    }
   ],
   "source": [
    "hyper_params = {\"lr\": 0.01,\n",
    "                        \"epoch\": 50,\n",
    "                        \"wordNgrams\": 2,\n",
    "                        \"dim\": 80\n",
    "              \n",
    "                        \n",
    "               }\n",
    "\n",
    "\n",
    "model.train_supervised(input=training_data_path, **hyper_params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'lr': 0.01, 'epoch': 50, 'wordNgrams': 2, 'dim': 80},accuracy:0.31875,validation:0.31875\n",
      "\n"
     ]
    }
   ],
   "source": [
    "        result = model.test(testing_data_path)\n",
    "        validation = model.test(testing_data_path)\n",
    "        \n",
    "        # DISPLAY ACCURACY OF TRAINED MODEL\n",
    "        text_line = str(hyper_params) + \",accuracy:\" + str(result[1])  + \",validation:\" + str(validation[1]) + '\\n' \n",
    "        print(text_line)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(160, 0.3125, 0.3125)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(160, 0.3125, 0.3125)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>my girlfriend wanted to travel to the moon i s...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>whats lantern up to these days its sad when a ...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>even likes are thanos marvelousjokes odd likes...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>cat memes meow  meme machine</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>no new baby godfather memes dies memegeneratornet</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5918</th>\n",
       "      <td>when u sneeze in front of ur pet and they look...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5937</th>\n",
       "      <td>the modi government has decided to include 15 ...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6010</th>\n",
       "      <td>world leader 10year challenge 2009 2019 even w...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6068</th>\n",
       "      <td>theres a dog behind you what does he look like</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6070</th>\n",
       "      <td>god creates jews god creates jesus jews kills ...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>800 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  \\\n",
       "1     my girlfriend wanted to travel to the moon i s...   \n",
       "2     whats lantern up to these days its sad when a ...   \n",
       "4     even likes are thanos marvelousjokes odd likes...   \n",
       "10                         cat memes meow  meme machine   \n",
       "13    no new baby godfather memes dies memegeneratornet   \n",
       "...                                                 ...   \n",
       "5918  when u sneeze in front of ur pet and they look...   \n",
       "5937  the modi government has decided to include 15 ...   \n",
       "6010  world leader 10year challenge 2009 2019 even w...   \n",
       "6068     theres a dog behind you what does he look like   \n",
       "6070  god creates jews god creates jesus jews kills ...   \n",
       "\n",
       "                           label  \n",
       "1         __label__not_offensive  \n",
       "2         __label__not_offensive  \n",
       "4         __label__not_offensive  \n",
       "10        __label__not_offensive  \n",
       "13        __label__not_offensive  \n",
       "...                          ...  \n",
       "5918  __label__hateful_offensive  \n",
       "5937  __label__hateful_offensive  \n",
       "6010  __label__hateful_offensive  \n",
       "6068  __label__hateful_offensive  \n",
       "6070  __label__hateful_offensive  \n",
       "\n",
       "[800 rows x 2 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"Length\"]= train[\"text\"].str.len() \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>Length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>my girlfriend wanted to travel to the moon i s...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>whats lantern up to these days its sad when a ...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>even likes are thanos marvelousjokes odd likes...</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "      <td>124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>cat memes meow  meme machine</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>no new baby godfather memes dies memegeneratornet</td>\n",
       "      <td>__label__not_offensive</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5918</th>\n",
       "      <td>when u sneeze in front of ur pet and they look...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5937</th>\n",
       "      <td>the modi government has decided to include 15 ...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6010</th>\n",
       "      <td>world leader 10year challenge 2009 2019 even w...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6068</th>\n",
       "      <td>theres a dog behind you what does he look like</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6070</th>\n",
       "      <td>god creates jews god creates jesus jews kills ...</td>\n",
       "      <td>__label__hateful_offensive</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>800 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  \\\n",
       "1     my girlfriend wanted to travel to the moon i s...   \n",
       "2     whats lantern up to these days its sad when a ...   \n",
       "4     even likes are thanos marvelousjokes odd likes...   \n",
       "10                         cat memes meow  meme machine   \n",
       "13    no new baby godfather memes dies memegeneratornet   \n",
       "...                                                 ...   \n",
       "5918  when u sneeze in front of ur pet and they look...   \n",
       "5937  the modi government has decided to include 15 ...   \n",
       "6010  world leader 10year challenge 2009 2019 even w...   \n",
       "6068     theres a dog behind you what does he look like   \n",
       "6070  god creates jews god creates jesus jews kills ...   \n",
       "\n",
       "                           label  Length  \n",
       "1         __label__not_offensive      94  \n",
       "2         __label__not_offensive     142  \n",
       "4         __label__not_offensive     124  \n",
       "10        __label__not_offensive      28  \n",
       "13        __label__not_offensive      49  \n",
       "...                          ...     ...  \n",
       "5918  __label__hateful_offensive     110  \n",
       "5937  __label__hateful_offensive     163  \n",
       "6010  __label__hateful_offensive     100  \n",
       "6068  __label__hateful_offensive      46  \n",
       "6070  __label__hateful_offensive     123  \n",
       "\n",
       "[800 rows x 3 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean1 = train['Length'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79.6975"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}