4-1.텍스트분류전처리.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"4-1.텍스트분류전처리.ipynb","provenance":[],"collapsed_sections":[],"mount_file_id":"1VqveTka8W1AQe3FnVqOpDNqgxYplVK5w","authorship_tag":"ABX9TyO0zfuw6PSg58F34It7wa6G"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"NecjZvQcicjM"},"source":["# 4.텍스트분류\n","##### 전처리, 모델링\n","\n","네이버 영화 데이터 긍 부정 분류 데이터  전처리 모델\n","\n","### EDA"]},{"cell_type":"code","metadata":{"id":"6xI60ONR1nKM"},"source":["import numpy as np \n","import pandas as pd\n","import os\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","from wordcloud import WordCloud\n","from sklearn.model_selection import train_test_split\n","import numpy as np\n","\n","%matplotlib inline"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GW2QbRhh5nFd"},"source":["DATA_IN_PATH ='/content/drive/MyDrive/2021-1/AI데이터활용교재개발/자연어처리(텐서플로,머신러닝)/code/data/'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"A0D8gcl_5uSy","executionInfo":{"status":"ok","timestamp":1619686345095,"user_tz":-540,"elapsed":362,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"7ec13214-6bef-485b-c977-5b8a94e71929"},"source":["# 파일 크기 확인\n","print(\"파일 크기 : \")\n","for file in os.listdir(DATA_IN_PATH):\n","    if 'txt' in file :\n","        print(file.ljust(30) + str(round(os.path.getsize(DATA_IN_PATH + file) / 1000000, 2)) + 'MB')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["파일 크기 : \n","ratings_train.txt             14.63MB\n","ratings_test.txt              4.89MB\n","ratings.txt                   19.52MB\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"o2F3PcyPe3Dl"},"source":["# 데이터 읽어오기\n","data = pd.read_csv(DATA_IN_PATH + 'ratings.txt', header = 0, delimiter = '\\t', quoting = 3)\n","train_data, test_data = train_test_split(data, test_size=0.25, random_state=1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":200},"id":"a7XhDGJ155yp","executionInfo":{"status":"ok","timestamp":1619686348783,"user_tz":-540,"elapsed":876,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"2873bb82-5770-4156-c6e8-cb28accba0d9"},"source":["train_data.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>document</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>76034</th>\n","      <td>6320715</td>\n","      <td>어린시절 집에 찾아가 낙하산인형 찾는 장면이나 면도하는 장면도 좋죠.</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>51652</th>\n","      <td>9280468</td>\n","      <td>영화의 마약빠는듯한 효과들이 좋았고 난 끝부분이 너무 싹둑잘라버리는 느낌만아니면 거...</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>138222</th>\n","      <td>6261395</td>\n","      <td>이땐 김사랑 별로네_-</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>115575</th>\n","      <td>3735990</td>\n","      <td>신고안한사람은 글타쳐도 상관없는사람까지 괴롭히냐 별미친귀신 다보겠네;;</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>45249</th>\n","      <td>2160536</td>\n","      <td>이레느야곱은최고다..</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["             id                                           document  label\n","76034   6320715             어린시절 집에 찾아가 낙하산인형 찾는 장면이나 면도하는 장면도 좋죠.      1\n","51652   9280468  영화의 마약빠는듯한 효과들이 좋았고 난 끝부분이 너무 싹둑잘라버리는 느낌만아니면 거...      1\n","138222  6261395                                       이땐 김사랑 별로네_-      0\n","115575  3735990            신고안한사람은 글타쳐도 상관없는사람까지 괴롭히냐 별미친귀신 다보겠네;;      0\n","45249   2160536                                        이레느야곱은최고다..      1"]},"metadata":{"tags":[]},"execution_count":51}]},{"cell_type":"code","metadata":{"id":"zhNOvIec6EsX","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1619686350977,"user_tz":-540,"elapsed":550,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"e38a0e04-b92c-4b3b-a84b-9e9adfac4a51"},"source":["print('전체 학습데이터의 개수: {}'.format(len(train_data)))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["전체 학습데이터의 개수: 150000\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Z5N0L6ZC6IBk","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1619686353204,"user_tz":-540,"elapsed":519,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"7dd88fa5-d4a5-4834-db25-8065505538bd"},"source":["# 데이터 수 확인\n","train_length = train_data['document'].astype(str).apply(len)\n","train_length.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["76034     38\n","51652     54\n","138222    12\n","115575    39\n","45249     11\n","Name: document, dtype: int64"]},"metadata":{"tags":[]},"execution_count":53}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":367},"id":"vNwAwM1X7wK4","executionInfo":{"status":"ok","timestamp":1619686362828,"user_tz":-540,"elapsed":2002,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"d491362f-b860-4a6b-9f7b-086e1280b28f"},"source":["# 그래프에 대한 이미지 사이즈 선언\n","# figsize: (가로, 세로) 형태의 튜플로 입력\n","plt.figure(figsize=(12, 5))\n","# 히스토그램 선언\n","# bins: 히스토그램 값들에 대한 버켓 범위\n","# range: x축 값의 범위\n","# alpha: 그래프 색상 투명도\n","# color: 그래프 색상\n","# label: 그래프에 대한 라벨\n","plt.hist(train_length, bins=200, alpha=0.5, color= 'r', label='word')\n","plt.yscale('log', nonposy='clip')\n","# 그래프 제목\n","plt.title('Log-Histogram of length of review')\n","# 그래프 x 축 라벨\n","plt.xlabel('Length of review')\n","# 그래프 y 축 라벨\n","plt.ylabel('Number of review')"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Text(0, 0.5, 'Number of review')"]},"metadata":{"tags":[]},"execution_count":54},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAtMAAAFNCAYAAADCcOOfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debgkdX3v8feHNQhkUFAj66CDKHHPEXFH3CAyYIgLuHMRNNf1RhMVUcYYjd7EJSQumQhiYgQUjWEEF4xsUZDNhU0UcWGVcRsRryD6vX9UjTbHs/SpOX26+5z363nOM91V1VXfqq45/enf+dWvUlVIkiRJmruNhl2AJEmSNK4M05IkSVJHhmlJkiSpI8O0JEmS1JFhWpIkSerIMC1JkiR1ZJiWtOgleXSSK4ddxzCl8cEkP0ly/hTzX5Dkf4ZU2/FJ/nae1rV7kq8muTnJy+djnTNs68gkHxjkNiSNPsO0pIFJ8t0kTxjwNvZOcu0U089M8kKAqjqnqnbvY12rknx4EHWOgEcBTwR2rKo9h1XEAoT2vwbOqKqtq+qYAW6HqnprVb1wkNuQNPoM05K0AJJsMuQSdgG+W1W3DLmOQdsFuKyfBdvWej8HJW0Qf4lIWnBJNk/y7iTXtz/vTrJ5z/y/TnJDO++FSSrJig3Y3h1ar5O8Jsl1bVeAK5M8Psm+wJHAM5P8PMnX2mW3T3JKkh8nuSrJ4T3r2SLJh9quE1e0dfdu57vttr4O3JJkkySvTfLtdtuXJ/mznuVfkOSLSd6V5KdJrk7yiHb6NUluSvL8GfZzylqTHAZ8AHh4u29v6uOY3SfJ6e26rkzyjJ55xyd5T5JT2/34cpJ79cx/UvuadUnem+Ss9n28L/D+njp+2rPJO0+3vilqOyDJZe0xOrNdL0m+ADwO+Od2/fee4rVnJnlLki8CvwDuOd2+JnlYkhuTbNzz+j9r38/f+0tGkr2SfKmt62tJ9m6nPy7JJT3LnZ7kgp7n5yR56mzviaTRZJiWNAyvB/YCHgQ8ENgTOAqgDbV/CTwBWAHsPZ8bTrI78FLgoVW1NfBkmhbbzwBvBU6qqq2q6oHtS04ErgW2B54GvDXJPu28o4HlwD1pulA8Z4pNHgI8Bdimqm4Hvg08GlgGvAn4cJJ79Cz/MODrwLbAR9rtP5TmWDyHJihuNc3uTVlrVR0LvBg4t923o2c5RlsCp7fbvxtwMPDeJHv0LHZwW/+dgauAt7Sv3Q44GXhduw9XAo8AqKorJtWxzWzrm6K2ewMnAK8E7gqcBqxJsllV7QOcA7y0Xf83p9nF5wJHAFsDa6fb16r6MnALsE/Pa5/VLju5rh2AU4G/Be4CvBr4eJK7AucBuyXZLsmmwAOA7ZNsnWQLYKKtW9IYMkxLGoZnA39TVTdV1VqaEPXcdt4zgA9W1WVV9QtgVR/r275tDfztD00f4an8Gtgc2CPJplX13ar69lQLJtkJeCTwmqr6ZVV9laaF93k9tb61qn5SVdcCU/XRPaaqrqmq/wdQVR+rquur6jdVdRLwLZovE+t9p6o+WFW/Bk4CdqI5VrdW1eeA22iC9VxrnYv9ab5gfLCqbq+qrwAfB57es8x/VtX57ReE/6D5YgTwp8BlVfWJdt4xwI19bHO69U32TODUqjq9qn4F/AOwBW1g79Px7fl1O7DvLPt6As0XIpJs3e7fCVOs8znAaVV1Wvveng5cCPxp+95fADwG+BPga8AXad6vvYBvVdWP5lC/pBFimJY0DNsD3+t5/r122vp51/TM++3jJDu3f77/eZKf9yxzfVVt0/sDTHmRW1VdRdOquQq4KcmJSbafatm2lh9X1c2Tat1htlqnm5bkeWlGm1gf+u8HbNezyA96Hq8P4JOnTdUyPVutc7EL8LBJX06eDfxRzzK9AfkXPTXd4ZhUVdG0ls9muvVNdodzp6p+025vLvvZ+57Mtq8fAQ5K0w3pIODiqvoev28X4OlTfKFb/1eHs2j+yvKY9vGZwGPbn7PmULukEWOYljQM19OEj/V2bqcB3ADs2DNvp/UPqur77Z/vt6qq6cLWrKrqI1X1qLaGAt6+ftYUdd6lbZHsrfW62Wrt3dz6B0l2Af6VppvJtm3ovxRIx12ZS61zcQ1w1qQvKFtV1V/08do7HJMk4Y7HaPIxnqs7nDvt+ndibvvZW8OM+1pVl9OE9/2YpotHz3r+fdJ6tqyqt7XzJ4fpszBMS4uCYVrSoG2a5A96fjah+TP5UUnu2vaxfSOw/kKujwKHJrlvkjsBb5jPYtKMQ7xP29L4S5qW3t+0s38ALE87wkNVXQN8Cfi7tvYHAIdNqvV1Se7c9pl96Syb35ImyK1tazmUpmV6g/VR61x8Crh3kucm2bT9eej6C/1mcSpw/yRPbd/rl3DHFu0fADsm2axDXdAc86ekuWh0U+BVwK00+95FP/v6EeAVNEH4Y9Os58PAyiRPTrJx+x7snWT9F4kvAbvTdOk5v6ouo20VB87uWLukEWCYljRop9EE1vU/q2gu0rqQ5kK7S4CL22lU1adp+tmeQXMh2nntem6dp3o2B94G/JCma8HdaC6Wg98FpR8lubh9fAjNRYbXA/8JHF1Vn2/n/Q1NF4bvAJ+nufBu2jrbVs53AOfShMr70/SdnS8z1dq3tqvIk2guxrue5ji9nebYzfbaH9L0N/6/wI+APWje6/XH5Qs0Q9fdmOSHHWq7kqZ/8j/RvIcrgZVVddtc19Wur599PYGmBfkL7f5NtZ5rgANpRoRZS9NS/Ve0n7PtkIQX0/QnX1/rucD3quqmLrVLGg1purNJ0mhqWwgvBTZvLxgbWUn+Aji4qh477FpGRdvKfy3w7Ko6Y9j1SNJ8s2Va0shpx/LdPMmdaVoJ14xikE5yjySPTLJRO+Teq2hahJe0tqvDNm1XmiNp+oSfN8vLJGksGaYljaIXATfRjMn8a6CfC9+GYTPgX4Cbabov/Bfw3qFWNBoeTvPere+G8dT1QwNK0mJjNw9JkiSpI1umJUmSpI4M05IkSVJHmwy7gA2x3Xbb1fLly4ddhiRJkha5iy666IdVddfJ08c6TC9fvpwLL7xw2GVIkiRpkUvyvamm281DkiRJ6mgsw3SSlUlWr1u3btilSJIkaQkbyzBdVWuq6ohly5YNuxRJkiQtYWMZpiVJkqRRYJiWJEmSOjJMS5IkSR0ZpiVJkqSODNOSJElSR2MZph0aT5IkSaNgLMO0Q+NJkiRpFIz17cS1AVatmvqxJEmS+jaWLdOSJEnSKLBleimZrgXaVmpJkqRODNOaniFbkiRpRnbzkCRJkjoayzDt0HiSJEkaBWPZzaOq1gBrJiYmDh92LUuGXT4kSZJ+z1iGaY2Qfi5qlCRJWqTGspuHJEmSNApsmdZg2C1EkiQtAYbpxWjUg6xdQyRJ0iJhmNboGPUvAZIkSZMYpjX6DNmSJGlEGaa1OBi4JUnSEBimNV4MzZIkaYSMZZhOshJYuWLFimGXolFk4JYkSQtkLMO0d0BU36YL1gZuSZI0D8YyTEvzymAtSZI6MkxL0zFkS5KkWRimFwvD3mB5oxlJkjQFw7Q0X2zJliRpyTFMS4NgsJYkaUnYaNgFSJIkSePKMC1JkiR1ZDcPadDs8iFJ0qJly7QkSZLUkS3T0rBMbqWea6u1Ld6SJA2dYVoaFd7uXJKksTOW3TySrEyyet26dcMuRZIkSUvYWLZMV9UaYM3ExMThw65FGnve3VGSpM7GMkxL6sCuI5IkzTvD9KjqJ/gYiDQVzwtJkhaMYXoc2KIoSZI0ksbyAkRJkiRpFBimJUmSpI7s5jFsduHQOPA8lSRpSrZMS5IkSR3ZMr1QbNmTJEladGyZliRJkjoyTEuSJEkd2c1jGOzmIUmStCjYMi1JkiR1ZJiWJEmSOjJMS5IkSR3ZZ3o+Te4Lbd9oSZKkRc0wLWluHDNdkqTfspuHJEmS1JFhWpIkSepoZMJ0kvsmeX+Sk5P8xbDrkSRJkmYz0D7TSY4D9gduqqr79UzfF/hHYGPgA1X1tqq6Anhxko2AfwPeN8ja5o19RiVJkpasQbdMHw/s2zshycbAe4D9gD2AQ5Ls0c47ADgVOG3AdUmSJEkbbKAt01V1dpLlkybvCVxVVVcDJDkROBC4vKpOAU5JcirwkUHWJmkeOLKHJGmJG8bQeDsA1/Q8vxZ4WJK9gYOAzZmhZTrJEcARADvvvPPgqpQkSdLslnjDysiMM11VZwJn9rHcamA1wMTERA22KkmSJGl6wxjN4zpgp57nO7bTJEmSpLEyjDB9AbBbkl2TbAYcDJwylxUkWZlk9bp16wZSoCRJktSPgYbpJCcA5wK7J7k2yWFVdTvwUuCzwBXAR6vqsrmst6rWVNURy5Ytm/+iJUmSpD4NejSPQ6aZfhoOfydJkqQxNzIXIEpaRJb4ld2SpKVjZG4nPhf2mZYkSdIoGMswbZ9pSZIkjYKxDNOSJEnSKDBMS5IkSR15AaKkwfJiREnSIjaWLdNegChJkqRRMJZh2gsQJUmSNArs5iFp4djlQ5K0yBimJQ2fIVuSNKYM0134YS9JkiTGNEwnWQmsXLFixbBLkTTfJn9Zna7V2i+1kqQRMJZhuqrWAGsmJiYOH3YtkiRJS44NGr81lmFakmylliSNgrEcGk+SJEkaBYZpSZIkqSO7eUgaf/1cpGhXEEnSAIxlmHY0D0lzZrCWJA3AWIZpR/OQNG+mC9YGbklSH+wzLUmSJHU0li3TkjQS7DoiSUueYVqSpmJQliR/F/bBbh6SJElSR7O2TCc5DDi7qr61APVI0niy9UaSlqR+unnsDPxLkuXARcDZwDlV9dUB1iVJo2nQQdlQLkljZdYwXVVHAyTZAjgc+Cvg3cDGgy1teo4zLWls9DP0nqFZ0ijxd9KczNpnOslRST4NfA5YAbwa2HHQhc2kqtZU1RHLli0bZhmSJEla4vrp5nEQcDtwKnAWcG5V3TrQqiRJM7de27ItSSNh1pbpqnoI8ATgfOCJwCVJ/mfQhUmSJEmjrp/RPO4HPBp4LDABXAOcM+C6JEld9NNibau2pMn8XdBZP9083kYTno8BLqiqXw22JEnSnPTzITjXD0oDtyT1pZ/RPPZvR/LY2SAtSUuQwVqSptXPaB4rga8Cn2mfPyjJKYMuTJIkSRp1/XTzWAXsCZwJUFVfTbLrAGuSJA2Drc6SNGf9hOlfVdW6JL3TakD19MWbtkjSmLPriKRFop8wfVmSZwEbJ9kNeDnwpcGWNbOqWgOsmZiYOHyYdUiSWoMIxwZuSWOgnzD9MuD1wK3ACcBngTcPsihJ0hgYRMA1NEuD4xfUgehnNI9f0ITp1w++HEnSouWHt6RFaNowneTdVfXKJGuYoo90VR0w0MokSZrNTLdZl6QFMFPL9L+3//7DQhQiSVok/FOyNDr8Pzhw04bpqrqofbgtcGpV3bowJUmSNED93GZdkvo0601bgJXAN5P8e5L9k/Rz0aIkSZK06PVzAeKhSTYF9gMOAd6T5PSqeuHAq5MkaS7sYiJpgfXVylxVv0ryaZoLEbcAngoYpiVJC2NDQ7LBWkuJ5/uCmjVMJ9kPeCawN80txT8APGOgVUmSNIps+ZY0ST8t088DTgJe5EWIkiRJ0u/002f6kCS7AI8GPp9kC2CTqrp54NVJkjRs/bRAO0KItGT1083jcOAI4C7AvYAdgfcDjx9saZIkLQF2HZHGWj/dPF4C7Al8GaCqvpXkbgOtahZJVgIrV6xYMcwyJEkaHEO2puJfQUZOP2H61qq6LQkA7TjTv3d78YVUVWuANRMTE4cPsw5J0iI16GBi8JEWjX7C9FlJjgS2SPJE4H8DawZbliRJi8woBmhbv0eL78dY6idMv4ZmTOlLgBcBp9EMjydJkkaFQWxx8f0cGzOG6SQbA5dV1X2Af12YkiRJ0rQWOmQZ6qQZzRimq+rXSa5MsnNVfX+hipIkST3mGmInL28gXjge6yWnn24edwYuS3I+cMv6iVV1wMCqkiRJC2sxhcBx2pdRr0+z6idMv2HgVUiSpNE3TiF1EDZk/5f6sVvE+rkD4lkLUYgkSZI0bvppmZYkSbqjQbS02nqrMWSYliRJw7OYQ/Ni3jf91rRhOsl/V9Xjk7y9ql6zkEVJkqRFYLqW5oUMmTONbDLddEOw5mCmlul7JHkEcECSE4H0zqyqiwdamSRJEgwu6BqaNQ9mCtNvpBnJY0fgnZPmFbDPoIqSJEljaqkEVFuy1Zo2TFfVycDJSd5QVW9ewJokSZLmZlAt1gZlzaKfofHenOQA4DHtpDOr6lODLUuSJGkKBl2NmI1mWyDJ3wGvAC5vf16R5K2DLkySJEkadf0MjfcU4EFV9RuAJB8CvgIcOcjCJEmSpFE3a8t0a5uex8sGUYgkSZI0bvppmf474CtJzqAZHu8xwGvnu5AkT6VpBf9D4Niq+tx8b0OSJEmaT7O2TFfVCcBewCeAjwMPr6qT+ll5kuOS3JTk0knT901yZZKrkry23c4nq+pw4MXAM+e6I5IkSdJC66ubR1XdUFWntD83zmH9xwP79k5IsjHwHmA/YA/gkCR79CxyVDtfkiRJGmn99pnupKrOBn48afKewFVVdXVV3QacCByYxtuBT3t3RUmSJI2DgYbpaewAXNPz/Np22suAJwBPS/Li6V6c5IgkFya5cO3atYOtVJIkSZrBjBcgtl0yLquq+wy6kKo6Bjimj+VWA6sBJiYmatB1SZIkSdOZsWW6qn4NXJlk53nc5nXATj3Pd2ynSZIkSWOln6Hx7gxcluR84Jb1E6vqgI7bvADYLcmuNCH6YOBZHdclSZIkDU0/YfoNXVee5ARgb2C7JNcCR1fVsUleCnwW2Bg4rqoum+N6VwIrV6xY0bU0SZIkaYPNGqar6qwkuwC7VdXnk9yJJgTPqqoOmWb6acBpc6r0jq9fA6yZmJg4vOs6JEmSpA0162geSQ4HTgb+pZ20A/DJQRYlSZIkjYN+hsZ7CfBI4GcAVfUt4G6DLEqSJEkaB/2E6Vvbm6sAkGQTYKhD0iVZmWT1unXrhlmGJEmSlrh+wvRZSY4EtkjyROBjwJrBljWzqlpTVUcsW7ZsmGVIkiRpiesnTL8WWAtcAryI5sLBowZZlCRJkjQO+hnN4zdJPgR8maZ7x5VV5Z0HJUmStOT1M5rHU4Bv09zq+5+Bq5LsN+jCZqnJPtOSJEkaun66ebwDeFxV7V1VjwUeB7xrsGXNzD7TkiRJGgX9hOmbq+qqnudXAzcPqB5JkiRpbEzbZzrJQe3DC5OcBnyUps/004ELFqA2SZIkaaTNdAHiyp7HPwAe2z5eC2wxsIokSZKkMTFtmK6qQxeyEEmSJGnczDo0XpJdgZcBy3uXr6oDBlfWrDWtBFauWLFiWCVIkiRJs4dp4JPAsTR3PfzNYMvpT1WtAdZMTEwcPuxaJEmStHT1E6Z/WVXHDLwSSZIkacz0E6b/McnRwOeAW9dPrKqLB1aVJEmSNAb6CdP3B54L7MPvunlU+1ySJElasvoJ008H7llVtw26GEmSJGmc9HMHxEuBbQZdyFwkWZlk9bp164ZdiiRJkpawfsL0NsA3knw2ySnrfwZd2Eyqak1VHbFs2bJhliFJkqQlrp9uHkcPvApJkiRpDM0apqvqrIUoRJIkSRo3/dwB8Waa0TsANgM2BW6pqj8cZGGSJEnSqOunZXrr9Y+TBDgQ2GuQRUmSJEnjoJ8LEH+rGp8EnjygeiRJkqSx0U83j4N6nm4ETAC/HFhFfUiyEli5YsWKYZYhSZKkJa6f0TxW9jy+HfguTVePoamqNcCaiYmJw4dZhyRJkpa2fvpMH7oQhUiSJEnjZtowneSNM7yuqurNA6hHkiRJGhsztUzfMsW0LYHDgG0Bw7QkSZKWtGnDdFW9Y/3jJFsDrwAOBU4E3jHd6yRJkqSlYsY+00nuAvwl8GzgQ8BDquonC1GYJEmSNOpm6jP998BBwGrg/lX18wWrSpIkSRoDM9205VXA9sBRwPVJftb+3JzkZwtTniRJkjS6ZuozPae7I0qSJElLzVgG5iQrk6xet27dsEuRJEnSEjaWYbqq1lTVEcuWLRt2KZIkSVrCxjJMS5IkSaPAMC1JkiR1ZJiWJEmSOjJMS5IkSR0ZpiVJkqSODNOSJElSR4ZpSZIkqSPDtCRJktTRtLcTlyRJkhbMqlVTPx5xtkxLkiRJHRmmJUmSpI4M05IkSVJHYxmmk6xMsnrdunXDLkWSJElL2FiG6apaU1VHLFu2bNilSJIkaQkbyzAtSZIkjQLDtCRJktSRYVqSJEnqyDAtSZIkdWSYliRJkjoyTEuSJEkdGaYlSZKkjgzTkiRJUkeGaUmSJKkjw7QkSZLUkWFakiRJ6sgwLUmSJHW0ybALkCRJ0hKyatXUj8eULdOSJElSR4ZpSZIkqSPDtCRJktSRYVqSJEnqaGTCdJJ7Jjk2ycnDrkWSJEnqx0DDdJLjktyU5NJJ0/dNcmWSq5K8FqCqrq6qwwZZjyRJkjSfBt0yfTywb++EJBsD7wH2A/YADkmyx4DrkCRJkubdQMN0VZ0N/HjS5D2Bq9qW6NuAE4EDB1mHJEmSNAjD6DO9A3BNz/NrgR2SbJvk/cCDk7xuuhcnOSLJhUkuXLt27aBrlSRJkqY1MndArKofAS/uY7nVwGqAiYmJGnRdkiRJ0nSG0TJ9HbBTz/Md22mSJEnSWBlGy/QFwG5JdqUJ0QcDz5rLCpKsBFauWLFiAOVJkiRpg61aNewKFsSgh8Y7ATgX2D3JtUkOq6rbgZcCnwWuAD5aVZfNZb1Vtaaqjli2bNn8Fy1JkiT1aaAt01V1yDTTTwNOG+S2JUmSpEEbmTsgSpIkSeNmLMN0kpVJVq9bt27YpUiSJGkJG8swbZ9pSZIkjYKxDNOSJEnSKDBMS5IkSR0ZpiVJkqSOxjJMewGiJEmSRsFYhmkvQJQkSdIoGMswLUmSJI0Cw7QkSZLUkWFakiRJ6mgsw7QXIEqSJGkUjGWY9gJESZIkjYKxDNOSJEnSKDBMS5IkSR0ZpiVJkqSODNOSJElSR2MZph3NQ5IkSaNgLMO0o3lIkiRpFIxlmJYkSZJGgWFakiRJ6sgwLUmSJHVkmJYkSZI6MkxLkiRJHRmmJUmSpI7GMkw7zrQkSZJGwViGaceZliRJ0igYyzAtSZIkjQLDtCRJktSRYVqSJEnqyDAtSZIkdWSYliRJkjoyTEuSJEkdGaYlSZKkjjYZdgFdJFkJrFyxYsWwS5EkSdIgrVo19eMRMZYt0960RZIkSaNgLMO0JEmSNAoM05IkSVJHhmlJkiSpI8O0JEmS1JFhWpIkSerIMC1JkiR1ZJiWJEmSOjJMS5IkSR0ZpiVJkqSODNOSJElSR4ZpSZIkqaNU1bBr6CzJWuB7C7Cp7YAfLsB2lgqP5/zyeM4vj+f88njOL4/n/PJ4zq/Ffjx3qaq7Tp441mF6oSS5sKomhl3HYuHxnF8ez/nl8ZxfHs/55fGcXx7P+bVUj6fdPCRJkqSODNOSJElSR4bp/qwedgGLjMdzfnk855fHc355POeXx3N+eTzn15I8nvaZliRJkjqyZVqSJEnqyDA9iyT7JrkyyVVJXjvsesZNkp2SnJHk8iSXJXlFO/0uSU5P8q323zsPu9ZxkWTjJF9J8qn2+a5Jvtyeoycl2WzYNY6TJNskOTnJN5JckeThnp/dJPk/7f/zS5OckOQPPD/nJslxSW5KcmnPtCnPxzSOaY/t15M8ZHiVj6Zpjufft//fv57kP5Ns0zPvde3xvDLJk4dT9eia6nj2zHtVkkqyXft8yZyfhukZJNkYeA+wH7AHcEiSPYZb1di5HXhVVe0B7AW8pD2GrwX+u6p2A/67fa7+vAK4ouf524F3VdUK4CfAYUOpanz9I/CZqroP8ECaY+v5OUdJdgBeDkxU1f2AjYGD8fycq+OBfSdNm+583A/Yrf05AnjfAtU4To7n94/n6cD9quoBwDeB1wG0n00HA3/cvua9bQ7Q7xzP7x9PkuwEPAn4fs/kJXN+GqZntidwVVVdXVW3AScCBw65prFSVTdU1cXt45tpgsoONMfxQ+1iHwKeOpwKx0uSHYGnAB9onwfYBzi5XcRjOQdJlgGPAY4FqKrbquqneH52tQmwRZJNgDsBN+D5OSdVdTbw40mTpzsfDwT+rRrnAdskucfCVDoepjqeVfW5qrq9fXoesGP7+EDgxKq6taq+A1xFkwPUmub8BHgX8NdA74V4S+b8NEzPbAfgmp7n17bT1EGS5cCDgS8Dd6+qG9pZNwJ3H1JZ4+bdNL+wftM+3xb4ac8Hg+fo3OwKrAU+2Had+UCSLfH8nLOqug74B5qWqRuAdcBFeH7Oh+nORz+jNtz/Aj7dPvZ4dpDkQOC6qvrapFlL5ngaprUgkmwFfBx4ZVX9rHdeNUPKOKzMLJLsD9xUVRcNu5ZFZBPgIcD7qurBwC1M6tLh+dmfth/vgTRfULYHtmSKPwdrw3g+zp8kr6fpivgfw65lXCW5E3Ak8MZh1zJMhumZXQfs1PN8x3aa5iDJpjRB+j+q6hPt5B+s/3NP++9Nw6pvjDwSOCDJd2m6HO1D0993m/bP6uA5OlfXAtdW1Zfb5yfThGvPz7l7AvCdqlpbVb8CPkFzznp+brjpzkc/ozpK8gJgf+DZ9bsxgj2ec3cvmi/QX2s/m3YELk7yRyyh42mYntkFwG7t1eib0VyYcMqQaxorbZ/eY4ErquqdPbNOAZ7fPn4+8F8LXdu4qarXVdWOVbWc5lz8QlU9GzgDeFq7mMdyDqrqRuCaJLu3kx4PXI7nZxffB/ZKcqf2//36Y+n5ueGmOx9PAZ7XjpqwF7CupzuIppFkX5rucgdU1S96Zp0CHJxk8yS70lw4d/4wahwXVXVJVd2tqpa3n03XAg9pf7cumfPTm7bMIsmf0vRT3Rg4rqreMuSSxkqSRwHnAJfwu36+R9L0m/4osDPwPeAZVTXVRQ2aQpK9gVdX1f5J7knTUn0X4CvAc6rq1mHWN06SPIjmgs7NgKuBQ2kaGjw/5yjJm4Bn0vzp/CvAC2n6SHp+9inJCcDewHbAD4CjgU8yxd+7IisAAAQSSURBVPnYfmn5Z5ruNL8ADq2qC4dR96ia5ni+Dtgc+FG72HlV9eJ2+dfT9KO+naZb4qcnr3Mpm+p4VtWxPfO/SzOizw+X0vlpmJYkSZI6spuHJEmS1JFhWpIkSerIMC1JkiR1ZJiWJEmSOjJMS5IkSR0ZpiVpHiX5+YDX/8r2rmMbvL12PN3PJ/lqkmfOT4W/XfcHkuwxn+uUpFG0yeyLSJJGyCuBD9OM27qhHgxQVQ+aaaEkm1TV7XNZcVW9cEMKk6RxYcu0JA1Yknsl+UySi5Kck+Q+7fTjkxyT5EtJrk7ytHb6Rknem+QbSU5PclqSpyV5ObA9cEaSM3rW/5YkX0tyXpK7T7H9uyT5ZJKvt8s8IMndaEL5Q9uW6XtNes2ZSd6d5ELgFUn+JMlZ7T58Nsk9ktwnyfk9r1me5JKe10+0j5+U5NwkFyf5WJKtkjw0ySfa+Qcm+X9JNkvyB0munue3QJIGxjAtSYO3GnhZVf0J8GrgvT3z7gE8CtgfeFs77SBgObAH8Fzg4QBVdQxwPfC4qnpcu+yWNHdweyBwNnD4FNt/E/CVqnoAzR1I/62qbqK5Q+E5VfWgqvr2FK/brKomgGOAfwKe1u7DccBbquobwGbtrZehufvhSb0rSLIdcBTwhKp6CHAh8Jc0d0Nc3yL+aOBS4KHAw2jukCpJY8FuHpI0QEm2Ah4BfKy5uy7Q3Mp4vU9W1W+Ay3talR8FfKydfmNvK/QUbgM+1T6+CHjiFMs8CvhzgKr6QpJtk/xhH+WvD8a7A/cDTm/3YWPghnbeR2lC9Nvafyf3vd6L5kvBF9vXbgacW1W3J/l2kvsCewLvBB7TrvucPmqTpJFgmJakwdoI+OkM/ZJv7XmcaZaZya+qqtrHv2Z+f6/f0v4b4LKqevgUy5xE80XhE0BV1bcmzQ9welUdMsVrzwb2A34FfB44niZM/9U81C5JC8JuHpI0QFX1M+A7SZ4OkMYDZ3nZF4E/b/tO3x3Yu2fezcDWcyzjHODZ7fb3Bn7Y1tWvK4G7Jnl4u45Nk/wxQNs95NfAG5jUxaN1HvDIJCva126Z5N49db2SpqV6LbAtTSv4pXPbPUkaHlumJWl+3SnJtT3P30kTZN+X5ChgU+BE4GszrOPjwOOBy4FrgIuBde281cBnklzf0296NquA45J8nWYUkOf3+ToAquq29uLIY5Iso/nseDdwWbvIScDfA7tO8dq1SV4AnJBkffeWo4Bv0vSNvjtNCzXA14E/6mlpl6SRF39nSdLoSbJVVf08ybbA+cAjq+rGYdclSbojW6YlaTR9Ksk2NBfsvdkgLUmjyZZpSZIkqSMvQJQkSZI6MkxLkiRJHRmmJUmSpI4M05IkSVJHhmlJkiSpI8O0JEmS1NH/BzN5+pG0m2nKAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 864x360 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xQokIjqH7x9T","executionInfo":{"status":"ok","timestamp":1619686364382,"user_tz":-540,"elapsed":1541,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"d98e64fa-2991-48d3-a144-6e40e8632543"},"source":["# 리뷰 길이 사분위수 확인\n","print('리뷰 길이 최대 값: {}'.format(np.max(train_length)))\n","print('리뷰 길이 최소 값: {}'.format(np.min(train_length)))\n","print('리뷰 길이 평균 값: {:.2f}'.format(np.mean(train_length)))\n","print('리뷰 길이 표준편차: {:.2f}'.format(np.std(train_length)))\n","print('리뷰 길이 중간 값: {}'.format(np.median(train_length)))\n","# 사분위의 대한 경우는 0~100 스케일로 되어있음\n","print('리뷰 길이 제 1 사분위: {}'.format(np.percentile(train_length, 25)))\n","print('리뷰 길이 제 3 사분위: {}'.format(np.percentile(train_length, 75)))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["리뷰 길이 최대 값: 146\n","리뷰 길이 최소 값: 1\n","리뷰 길이 평균 값: 35.24\n","리뷰 길이 표준편차: 29.58\n","리뷰 길이 중간 값: 27.0\n","리뷰 길이 제 1 사분위: 16.0\n","리뷰 길이 제 3 사분위: 42.0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Zlns4KfT74zu","executionInfo":{"status":"ok","timestamp":1620654667168,"user_tz":-540,"elapsed":1234,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}}},"source":["train_review = [review for review in train_data['document'] if type(review) is str]\n","test_review = [review for review in test_data['document'] if type(review) is str]\n"],"execution_count":1,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":298},"id":"cdBv1cTj7-Di","executionInfo":{"status":"ok","timestamp":1619683849428,"user_tz":-540,"elapsed":3533,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"25dd752c-6a28-4d36-dbeb-6fde66c7b1dd"},"source":["# 라벨 분포 확인 라벨 분포 확인\n","fig, axe = plt.subplots(ncols=1)\n","fig.set_size_inches(6, 3)\n","sns.countplot(train_data['label'])\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n","  FutureWarning\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["<matplotlib.axes._subplots.AxesSubplot at 0x7ff351f0be10>"]},"metadata":{"tags":[]},"execution_count":13},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAZEAAADQCAYAAADGUlwqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAARhklEQVR4nO3dfayedX3H8feHp4kPSJGzDilYsjU6ZBNpA1UXMyWWwjbLjBKf1o4RawI6zZZtuD/WDeeimc6BDyRkVlrndIhzMIN2TX2Ki1UOijxKqCijBGilSH0IOtx3f9y/M27raXtzles+Pee8X8mdc13f63dd9+9OTvLJ9fT7paqQJKmLQ2a6A5Kk2csQkSR1ZohIkjozRCRJnRkikqTODBFJUmeHzXQHxu3YY4+txYsXz3Q3JGnWuOGGG75XVRPTbZt3IbJ48WImJydnuhuSNGskuXtv27ycJUnqzBCRJHVmiEiSOjNEJEmdGSKSpM7m3dNZB2rpn22c6S7oIHTD36+e6S5IM8IQkeaQ/77kN2a6CzoInfhXN/d2bC9nSZI6M0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLUWW8hkuTZSW4c+uxO8tYkxyTZnOTO9ndBa58klyXZluSmJKcNHWtNa39nkjVD9aVJbm77XJYkff0eSdIv6i1EquqOqjq1qk4FlgI/Bj4FXAxsqaolwJa2DnA2sKR91gKXAyQ5BlgHnAGcDqybCp7W5g1D+63s6/dIkn7RuC5nnQl8u6ruBlYBG1p9A3BuW14FbKyBrcDRSY4DzgI2V9WuqnoI2AysbNuOqqqtVVXAxqFjSZLGYFwh8mrgY215YVXd15bvBxa25eOBe4b22d5q+6pvn6b+C5KsTTKZZHLnzp0H8jskSUN6D5EkRwAvBz6x57Z2BlF996GqrqiqZVW1bGJi2hkeJUkdjONM5Gzg61X1QFt/oF2Kov3d0er3AicM7beo1fZVXzRNXZI0JuMIkdfw2KUsgGuBqSes1gDXDNVXt6e0lgMPt8tem4AVSRa0G+orgE1t2+4ky9tTWauHjiVJGoNeR/FN8hTgZcAbh8rvBK5KcgFwN3Beq18HnANsY/Ak1/kAVbUryduB61u7S6pqV1u+ELgSOBL4TPtIksak1xCpqh8Bz9ij9iCDp7X2bFvARXs5znpg/TT1SeCUJ6SzkqTHzTfWJUmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ72GSJKjk1yd5FtJbk/yAudYl6S5o+8zkUuBz1bVc4DnAbfjHOuSNGf0FiJJng68GPgQQFX9tKq+j3OsS9Kc0eeZyEnATuDDSb6R5J/a/CJjn2NdktSPPkPkMOA04PKqej7wIx67dAWMb471JGuTTCaZ3LlzZ99fJ0nzRp8hsh3YXlVfbetXMwiVsc+xXlVXVNWyqlo2MTFxQD9KkvSY3kKkqu4H7kny7FY6E7gN51iXpDmj1+lxgTcDH01yBHAXg3nTD8E51iVpTuh7jvUbgWXTbHKOdUmaA3xjXZLUmSEiSerMEJEkdWaISJI6M0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLUmSEiSerMEJEkdWaISJI66zVEknw3yc1Jbkwy2WrHJNmc5M72d0GrJ8llSbYluSnJaUPHWdPa35lkzVB9aTv+trZv+vw9kqSfN44zkZdU1alVNTWa78XAlqpaAmzhsdkOzwaWtM9a4HIYhA6wDjgDOB1YNxU8rc0bhvZb2f/PkSRNmYnLWauADW15A3DuUH1jDWwFjm4zH54FbK6qXVX1ELAZWNm2HVVVW9sw8huHjiVJGoO+Q6SA/0xyQ5K1rbawzUoIcD+wsC0fD9wztO/2VttXffs0dUnSmPQ9s+FvVdW9SX4Z2JzkW8Mbq6qSVM99oAXYWoATTzyx76+TpHmj1zORqrq3/d0BfIrBPY0H2qUo2t8drfm9wAlDuy9qtX3VF01Tn64fV1TVsqpaNjExcaA/S5LU9BYiSZ6S5GlTy8AK4BbgWmDqCas1wDVt+VpgdXtKaznwcLvstQlYkWRBu6G+AtjUtu1Osrw9lbV66FiSpDHo83LWQuBT7anbw4B/qarPJrkeuCrJBcDdwHmt/XXAOcA24MfA+QBVtSvJ24HrW7tLqmpXW74QuBI4EvhM+0iSxqS3EKmqu4DnTVN/EDhzmnoBF+3lWOuB9dPUJ4FTDrizkqROfGNdktSZISJJ6swQkSR1ZohIkjozRCRJnY0UIkm2jFKTJM0v+3zEN8mTgCcDx7YX/aaGWj8Kx6mSpHlvf++JvBF4K/BM4AYeC5HdwPt77JckaRbYZ4hU1aXApUneXFXvG1OfJEmzxEhvrFfV+5K8EFg8vE9VbeypX5KkWWCkEEnyEeBXgRuBn7Xy1ERQkqR5atSxs5YBJ7fxrSRJAkZ/T+QW4Fe6fEGSQ5N8I8mn2/pJSb6aZFuSf01yRKv/Ulvf1rYvHjrG21r9jiRnDdVXttq2JBfv+d2SpH6NGiLHArcl2ZTk2qnPiPu+Bbh9aP1dwHur6teAh4ALWv0C4KFWf29rR5KTgVcDzwVWAh9swXQo8AHgbOBk4DWtrSRpTEa9nPXXXQ6eZBHwO8A7gD9pk0e9FHhta7KhHftyYNXQ91wNvL+1XwV8vKp+AnwnyTYGMyQCbGtDzpPk463tbV36Kkl6/EZ9OuuLHY//j8CfA09r688Avl9Vj7b17Tz20uLxwD3t+x5N8nBrfzywdeiYw/vcs0f9jOk64RzrktSPUYc9+UGS3e3zSJKfJdm9n31+F9hRVTc8IT09AM6xLkn9GPVMZOpMgqFLTMv3s9uLgJcnOQd4EoOhUi4Fjk5yWDsbWQTc29rfC5wAbE9yGPB04MGh+pThffZWlySNweMexbcG/h04az/t3lZVi6pqMYMb45+rqtcBnwde2ZqtAa5py9e2ddr2z7VHiq8FXt2e3joJWAJ8jcGc60va015HtO8Y9Wa/JOkJMOrLhq8YWj2EwXsjj3T8zr8APp7kb4FvAB9q9Q8BH2k3zncxCAWq6tYkVzG4Yf4ocFFV/az1603AJuBQYH1V3dqxT5KkDkZ9Ouv3hpYfBb7L4JLWSKrqC8AX2vJdPPZ01XCbR4BX7WX/dzB4wmvP+nXAdaP2Q5L0xBr1nsj5fXdEkjT7jPp01qIkn0qyo30+2d4BkSTNY6PeWP8wg5vWz2yf/2g1SdI8NmqITFTVh6vq0fa5EvCFC0ma50YNkQeTvH5qzKokr2fwDockaR4bNUT+CDgPuB+4j8F7HH/YU58kSbPEqI/4XgKsqaqHAJIcA7ybQbhIkuapUc9EfnMqQACqahfw/H66JEmaLUYNkUOSLJhaaWcio57FSJLmqFGD4D3AV5J8oq2/imneIJckzS+jvrG+MckkgwmlAF5RVU7+JEnz3Mij+FbVbVX1/vbZb4AkeVKSryX5ZpJbk/xNqzvHuiTNEY97KPjH4SfAS6vqecCpwMoky3GOdUmaM3oLkTbvyA/b6uHtUwwuiV3d6huAc9vyqrZO237mnnOsV9V3gKk51k+nzbFeVT8FpuZYlySNSZ9nIrQzhhuBHcBm4NuMOMc6MDzH+p5zqR+/j7okaUx6DZGq+llVncpg6trTgef0+X17k2Rtkskkkzt37pyJLkjSnNRriEypqu8zmBb3BbQ51tum6eZYZ8Q51vc19/qe339FVS2rqmUTE44bKUlPlN5CJMlEkqPb8pHAy4DbcY51SZoz+nzr/DhgQ3uK6hDgqqr6dJLbcI51SZoTeguRqrqJacbXco51SZo7xnJPRJI0NxkikqTODBFJUmeGiCSpM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSpsz6Hgj8hyeeT3Jbk1iRvafVjkmxOcmf7u6DVk+SyJNuS3JTktKFjrWnt70yyZqi+NMnNbZ/L2nS6kqQx6fNM5FHgT6vqZGA5cFGSk4GLgS1VtQTY0tYBzmYwV8gSYC1wOQxCB1gHnMFg9N91U8HT2rxhaL+VPf4eSdIeeguRqrqvqr7eln/AYEKq44FVwIbWbANwblteBWysga0MZkA8DjgL2FxVu6rqIQZzta9s246qqq1t8qqNQ8eSJI3BWO6JJFnMYG6RrwILq+q+tul+YGFbPh64Z2i37a22r/r2aerTfb9zrEtSD3oPkSRPBT4JvLWqdg9va2cQ1XcfnGNdkvrRa4gkOZxBgHy0qv6tlR9ol6Jof3e0+r3ACUO7L2q1fdUXTVOXJI1Jn09nhcG86bdX1T8MbboWmHrCag1wzVB9dXtKaznwcLvstQlYkWRBu6G+AtjUtu1Osrx91+qhY0mSxqC3OdaBFwF/ANyc5MZW+0vgncBVSS4A7gbOa9uuA84BtgE/Bs4HqKpdSd4OXN/aXVJVu9ryhcCVwJHAZ9pHkjQmvYVIVX0Z2Nt7G2dO076Ai/ZyrPXA+mnqk8ApB9BNSdIB8I11SVJnhogkqTNDRJLUmSEiSerMEJEkdWaISJI6M0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLUmSEiSeqsz6Hg1yfZkeSWodoxSTYnubP9XdDqSXJZkm1Jbkpy2tA+a1r7O5OsGaovTXJz2+eyNhy8JGmM+jwTuRJYuUftYmBLVS0BtrR1gLOBJe2zFrgcBqEDrAPOAE4H1k0FT2vzhqH99vwuSVLPeguRqvoSsGuP8ipgQ1veAJw7VN9YA1uBo9ush2cBm6tqV1U9BGwGVrZtR1XV1jaE/MahY0mSxmTc90QWthkJAe4HFrbl44F7htptb7V91bdPU59WkrVJJpNM7ty588B+gSTp/83YjfV2BlFj+q4rqmpZVS2bmJgYx1dK0rww7hB5oF2Kov3d0er3AicMtVvUavuqL5qmLkkao3GHyLXA1BNWa4Brhuqr21Nay4GH22WvTcCKJAvaDfUVwKa2bXeS5e2prNVDx5IkjUlvc6wn+Rjw28CxSbYzeMrqncBVSS4A7gbOa82vA84BtgE/Bs4HqKpdSd4OXN/aXVJVUzfrL2TwBNiRwGfaR5I0Rr2FSFW9Zi+bzpymbQEX7eU464H109QngVMOpI+SpAPjG+uSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKmzWR8iSVYmuaPNtX7x/veQJD1RZnWIJDkU+ACDOdpPBl6T5OSZ7ZUkzR+zOkSA04FtVXVXVf0U+DiD+dolSWMw20Nkb3OwS5LGoLf5RA4mSdYCa9vqD5PcMZP9mUOOBb430504GOTda/bfSOPm/+eUdTnQIzxrbxtme4jsbQ72n1NVVwBXjKtT80WSyapaNtP9kKbj/+d4zPbLWdcDS5KclOQI4NUM5muXJI3BrD4TqapHk7wJ2AQcCqyvqltnuFuSNG/M6hABqKrrgOtmuh/zlJcIdTDz/3MMUlUz3QdJ0iw12++JSJJmkCGiThxuRgerJOuT7Ehyy0z3ZT4wRPS4OdyMDnJXAitnuhPzhSGiLhxuRgetqvoSsGum+zFfGCLqwuFmJAGGiCTpABgi6mKk4WYkzX2GiLpwuBlJgCGiDqrqUWBquJnbgascbkYHiyQfA74CPDvJ9iQXzHSf5jLfWJckdeaZiCSpM0NEktSZISJJ6swQkSR1ZohIkjozRKQeJfnhfrYvfryjzSa5MskrD6xn0hPDEJEkdWaISGOQ5KlJtiT5epKbkwyPenxYko8muT3J1Ume3PZZmuSLSW5IsinJcTPUfWmvDBFpPB4Bfr+qTgNeArwnSdq2ZwMfrKpfB3YDFyY5HHgf8MqqWgqsB94xA/2W9umwme6ANE8E+LskLwb+l8HQ+Qvbtnuq6r/a8j8Dfwx8FjgF2Nyy5lDgvrH2WBqBISKNx+uACWBpVf1Pku8CT2rb9hx7qBiEzq1V9YLxdVF6/LycJY3H04EdLUBeAjxraNuJSabC4rXAl4E7gImpepLDkzx3rD2WRmCISOPxUWBZkpuB1cC3hrbdAVyU5HZgAXB5m3b4lcC7knwTuBF44Zj7LO2Xo/hKkjrzTESS1JkhIknqzBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKmz/wM1YPpyPgFxxQAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x216 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WXz6O05X8GwS","executionInfo":{"status":"ok","timestamp":1619683849430,"user_tz":-540,"elapsed":3521,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"003e02c5-2c86-4813-d5f5-675d81106fc0"},"source":["print(\"긍정 리뷰 개수: {}\".format(train_data['label'].value_counts()[1]))\n","print(\"부정 리뷰 개수: {}\".format(train_data['label'].value_counts()[0]))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["긍정 리뷰 개수: 74815\n","부정 리뷰 개수: 75185\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"f_j2Dnu09vpM","executionInfo":{"status":"ok","timestamp":1619683849673,"user_tz":-540,"elapsed":3749,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"305b4ff4-04bb-4b4c-cdf5-8e65560769ef"},"source":["train_data['document'] = train_data['document'].astype(str)\n","train_word_counts = train_data['document'].apply(lambda x:len(x.split(' ')))\n","print('리뷰 단어 개수 최대 값: {}'.format(np.max(train_word_counts)))\n","print('리뷰 단어 개수 최소 값: {}'.format(np.min(train_word_counts)))\n","print('리뷰 단어 개수 평균 값: {:.2f}'.format(np.mean(train_word_counts)))\n","print('리뷰 단어 개수 표준편차: {:.2f}'.format(np.std(train_word_counts)))\n","print('리뷰 단어 개수 중간 값: {}'.format(np.median(train_word_counts)))\n","# 사분위의 대한 경우는 0~100 스케일로 되어있음\n","print('리뷰 단어 개수 제 1 사분위: {}'.format(np.percentile(train_word_counts, 25)))\n","print('리뷰 단어 개수 제 3 사분위: {}'.format(np.percentile(train_word_counts, 75)))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["리뷰 단어 개수 최대 값: 40\n","리뷰 단어 개수 최소 값: 1\n","리뷰 단어 개수 평균 값: 7.59\n","리뷰 단어 개수 표준편차: 6.52\n","리뷰 단어 개수 중간 값: 6.0\n","리뷰 단어 개수 제 1 사분위: 3.0\n","리뷰 단어 개수 제 3 사분위: 9.0\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  \"\"\"Entry point for launching an IPython kernel.\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WRumoG_B9ylN","executionInfo":{"status":"ok","timestamp":1619683849957,"user_tz":-540,"elapsed":3973,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"32e10b80-291d-476e-a3c3-6dfc9adc6d19"},"source":["qmarks = np.mean(train_data['document'].astype(str).apply(lambda x: '?' in x)) # 물음표가 구두점으로 쓰임\n","fullstop = np.mean(train_data['document'].astype(str).apply(lambda x: '.' in x)) # 마침표\n","                  \n","print('물음표가있는 질문: {:.2f}%'.format(qmarks * 100))\n","print('마침표가 있는 질문: {:.2f}%'.format(fullstop * 100))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["물음표가있는 질문: 8.27%\n","마침표가 있는 질문: 51.81%\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"UluYvI7591S1"},"source":["## 데이터 전처리"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JvVGVoFd90xO","executionInfo":{"status":"ok","timestamp":1619683855057,"user_tz":-540,"elapsed":9057,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"3f618456-7568-45ce-e481-6f8a02859a0d"},"source":["!pip install konlpy\n","import numpy as np\n","import pandas as pd\n","import re\n","import json\n","from konlpy.tag import Komoran\n","from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n","from tensorflow.python.keras.preprocessing.text import Tokenizer"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: konlpy in /usr/local/lib/python3.7/dist-packages (0.5.2)\n","Requirement already satisfied: JPype1>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (1.2.1)\n","Requirement already satisfied: lxml>=4.1.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (4.2.6)\n","Requirement already satisfied: tweepy>=3.7.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (3.10.0)\n","Requirement already satisfied: colorama in /usr/local/lib/python3.7/dist-packages (from konlpy) (0.4.4)\n","Requirement already satisfied: numpy>=1.6 in /usr/local/lib/python3.7/dist-packages (from konlpy) (1.19.5)\n","Requirement already satisfied: beautifulsoup4==4.6.0 in /usr/local/lib/python3.7/dist-packages (from konlpy) (4.6.0)\n","Requirement already satisfied: typing-extensions; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from JPype1>=0.7.0->konlpy) (3.7.4.3)\n","Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from tweepy>=3.7.0->konlpy) (1.3.0)\n","Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tweepy>=3.7.0->konlpy) (1.15.0)\n","Requirement already satisfied: requests[socks]>=2.11.1 in /usr/local/lib/python3.7/dist-packages (from tweepy>=3.7.0->konlpy) (2.23.0)\n","Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->tweepy>=3.7.0->konlpy) (3.1.0)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (2020.12.5)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (2.10)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (3.0.4)\n","Requirement already satisfied: PySocks!=1.5.7,>=1.5.6; extra == \"socks\" in /usr/local/lib/python3.7/dist-packages (from requests[socks]>=2.11.1->tweepy>=3.7.0->konlpy) (1.7.1)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ilXkRH4195Tp","executionInfo":{"status":"ok","timestamp":1619683855383,"user_tz":-540,"elapsed":9359,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"4083fb28-ebbb-464c-f585-e5cfe06311e3"},"source":["train_data = pd.read_csv(DATA_IN_PATH + 'ratings_train.txt', header=0, delimiter='\\t', quoting=3 )\n","print(train_data.head())"],"execution_count":null,"outputs":[{"output_type":"stream","text":["         id                                           document  label\n","0   9976970                                아 더빙.. 진짜 짜증나네요 목소리      0\n","1   3819312                  흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나      1\n","2  10265843                                  너무재밓었다그래서보는것을추천한다      0\n","3   9045019                      교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정      0\n","4   6483659  사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...      1\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hNzjgDSZ-Cf9","executionInfo":{"status":"ok","timestamp":1619683855384,"user_tz":-540,"elapsed":9344,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"b78d5385-6fd4-4d84-86b7-5fdd0ac85132"},"source":["# 특수문자 제거\n","review_text = re.sub(\"[^가-힣ㄱ-ㅎㅏ-ㅣ\\\\s]\", \"\", train_data['document'][0]) \n","print(review_text)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["아 더빙 진짜 짜증나네요 목소리\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"TU1NLiT9-EfX","executionInfo":{"status":"ok","timestamp":1619683862173,"user_tz":-540,"elapsed":16121,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"6da1ff79-df4c-4e9b-fdf9-0351cfa9da83"},"source":["# 형태소 분석\n","komoran=Komoran()\n","review_text = komoran.morphs(review_text)\n","print(review_text)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["['아', '더빙', '진짜', '짜증', '나', '네요', '목소리']\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"D8bX1g0A-F2E","executionInfo":{"status":"ok","timestamp":1619683862174,"user_tz":-540,"elapsed":16111,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"08915a4c-fc27-42c4-bb40-c0f84512d493"},"source":["# stopwords제거\n","stop_words = set(['은', '는', '이', '가', '하', '아', '것', '들','의', '있', '되', '수', '보', '주', '등', '한'])\n","clean_review = [token for token in review_text if not token in stop_words]\n","print(clean_review)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["['더빙', '진짜', '짜증', '나', '네요', '목소리']\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"WH1wmokY-LBV"},"source":["# 위 과정을 한번에 하는 함수\n","def preprocessing(review, komoran, remove_stopwords = False, stop_words = []):\n","    # 함수의 인자는 다음과 같다.\n","    # review : 전처리할 텍스트\n","    # komoran : komoran 객체를 반복적으로 생성하지 않고 미리 생성후 인자로 받는다.\n","    # remove_stopword : 불용어를 제거할지 선택 기본값은 False\n","    # stop_word : 불용어 사전은 사용자가 직접 입력해야함 기본값은 비어있는 리스트\n","    \n","    # 1. 한글 및 공백을 제외한 문자 모두 제거.\n","    review_text = re.sub(\"[^가-힣ㄱ-ㅎㅏ-ㅣ\\\\s]\", \"\", review)\n","    \n","    # 2. komoran 객체를 활용해서 형태소 단위로 나눈다.\n","    try:\n","        word_review = komoran.morphs(review_text)\n","    except:\n","        return []\n","        \n","    if remove_stopwords:\n","        \n","        # 불용어 제거(선택적)\n","        word_review = [token for token in word_review if not token in stop_words]\n","        \n","   \n","    return word_review"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"9xguwBev-O5_"},"source":["clean_train_review = []\n","clean_test_review = []\n","\n","\n","for review in train_data['document']:\n","    # 비어있는 데이터에서 멈추지 않도록 string인 경우만 진행\n","    if type(review) == str:\n","        clean_train_review.append(preprocessing(review, komoran, remove_stopwords = True, stop_words=stop_words))\n","    else:\n","        clean_train_review.append([])  #string이 아니면 비어있는 값 추가\n","\n","for review in test_data['document']:\n","    # 비어있는 데이터에서 멈추지 않도록 string인 경우만 진행\n","    if type(review) == str:\n","        clean_test_review.append(preprocessing(review, komoran, remove_stopwords = True, stop_words=stop_words))\n","    else:\n","        clean_test_review.append([])  #string이 아니면 비어있는 값 추가"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"KOteQjCJjLXC"},"source":["Tesorflow모델에 넣기 알맞은 형태로 데이터 가공"]},{"cell_type":"code","metadata":{"id":"NcIEpvvV-Slw"},"source":["tokenizer = Tokenizer()\n","tokenizer.fit_on_texts(clean_train_review)\n","train_sequences = tokenizer.texts_to_sequences(clean_train_review)\n","test_sequences = tokenizer.texts_to_sequences(clean_test_review)\n","\n","word_vocab = tokenizer.word_index # 단어 사전 형태\n","word_vocab[\"<PAD>\"] = 0"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"jUe9No7kjQSZ"},"source":["최대 길이 8로 데이터를 자르거나 padding"]},{"cell_type":"code","metadata":{"id":"59KItggN-Two"},"source":["MAX_SEQUENCE_LENGTH = 8 # 단어의 평균 갯수\n","\n","train_inputs = pad_sequences(train_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post') # 학습 데이터를 벡터화\n","train_labels = np.array(train_data['label']) # 학습 데이터의 라벨\n","\n","test_inputs = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post') # 테스트 데이터를 벡터화\n","test_labels = np.array(test_data['label']) # 테스트 데이터의 라벨"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t4wO1v5ytO_9","executionInfo":{"status":"ok","timestamp":1619686926446,"user_tz":-540,"elapsed":1155,"user":{"displayName":"이지현학부생","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggr12OZZ57xwrjXhR1pFDsgoInzIKHPWMT2FnsR=s64","userId":"08326129861451236666"}},"outputId":"84cfe40a-3e16-49e9-d4f3-c391fe4c6288"},"source":["len(train_labels)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["150000"]},"metadata":{"tags":[]},"execution_count":65}]},{"cell_type":"markdown","metadata":{"id":"w4U1_HoIjWcS"},"source":["전처리한 데이터 저장"]},{"cell_type":"code","metadata":{"id":"yJ7dLrzz-WOG"},"source":["TRAIN_INPUT_DATA = 'nsmc_train_input.npy'\n","TRAIN_LABEL_DATA = 'nsmc_train_label.npy'\n","TEST_INPUT_DATA = 'nsmc_test_input.npy'\n","TEST_LABEL_DATA = 'nsmc_test_label.npy'\n","DATA_CONFIGS = 'data_configs.json'\n","\n","data_configs = {}\n","\n","data_configs['vocab'] = word_vocab\n","data_configs['vocab_size'] = len(word_vocab) # vocab size 추가\n","\n","import os\n","# 저장하는 디렉토리가 존재하지 않으면 생성\n","if not os.path.exists(DATA_IN_PATH):\n","    os.makedirs(DATA_IN_PATH)\n","\n","# 전처리 된 학습 데이터를 넘파이 형태로 저장\n","np.save(open(DATA_IN_PATH + TRAIN_INPUT_DATA, 'wb'), train_inputs)\n","np.save(open(DATA_IN_PATH + TRAIN_LABEL_DATA, 'wb'), train_labels)\n","# 전처리 된 테스트 데이터를 넘파이 형태로 저장\n","np.save(open(DATA_IN_PATH + TEST_INPUT_DATA, 'wb'), test_inputs)\n","np.save(open(DATA_IN_PATH + TEST_LABEL_DATA, 'wb'), test_labels)\n","\n","# 데이터 사전을 json 형태로 저장\n","json.dump(data_configs, open(DATA_IN_PATH + DATA_CONFIGS, 'w'), ensure_ascii=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"VPitVbyqtq6d"},"source":["clean_train_df = pd.DataFrame({'review' : clean_train_review, 'sentiment' : train_labels})\n","clean_test_df = pd.DataFrame({'review' : clean_test_review, 'sentiment' : test_labels})\n","\n","\n","clean_train_df.to_csv(DATA_IN_PATH + \"clean_nsmc_train.csv\", index = False, encoding='UTF-8')\n","clean_test_df.to_csv(DATA_IN_PATH + \"clean_nsmc_test.csv\", index = False, encoding='UTF-8')\n","pd.concat([clean_train_df, clean_test_df]).to_csv(DATA_IN_PATH + \"clean_nsmc.csv\", index = False, encoding='UTF-8')"],"execution_count":null,"outputs":[]}]}