7.1. seoul-120-LDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rz2aYwDao1Yk"
   },
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/pytextbook/pytextbook/blob/main/7.1.%20seoul-120-LDA.ipynb) \n",
    "\n",
    "## 토픽 모델링\n",
    "* 실습을 위해 pyLDAvis 설치 \n",
    "* colab사용시 설치 후에도 제대로 동작하지 않거나 오류가 나면 런타임 재실행!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 27307,
     "status": "ok",
     "timestamp": 1672568395318,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "qcdv-HRKo1Yp",
    "outputId": "b1763362-36bc-4aa9-d4c5-861b58c469b7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[?25l\r",
      "\u001b[K     |▏                               | 10 kB 19.3 MB/s eta 0:00:01\r",
      "\u001b[K     |▍                               | 20 kB 15.5 MB/s eta 0:00:01\r",
      "\u001b[K     |▋                               | 30 kB 20.6 MB/s eta 0:00:01\r",
      "\u001b[K     |▉                               | 40 kB 7.7 MB/s eta 0:00:01\r",
      "\u001b[K     |█                               | 51 kB 7.9 MB/s eta 0:00:01\r",
      "\u001b[K     |█▏                              | 61 kB 9.2 MB/s eta 0:00:01\r",
      "\u001b[K     |█▍                              | 71 kB 8.9 MB/s eta 0:00:01\r",
      "\u001b[K     |█▋                              | 81 kB 9.4 MB/s eta 0:00:01\r",
      "\u001b[K     |█▉                              | 92 kB 10.3 MB/s eta 0:00:01\r",
      "\u001b[K     |██                              | 102 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██▏                             | 112 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██▍                             | 122 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██▋                             | 133 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██▊                             | 143 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███                             | 153 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███▏                            | 163 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███▍                            | 174 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███▋                            | 184 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███▊                            | 194 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████                            | 204 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████▏                           | 215 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████▍                           | 225 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████▋                           | 235 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████▊                           | 245 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████                           | 256 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▏                          | 266 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▍                          | 276 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▌                          | 286 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████▊                          | 296 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████                          | 307 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▏                         | 317 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▍                         | 327 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▌                         | 337 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████▊                         | 348 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████                         | 358 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████▏                        | 368 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████▎                        | 378 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████▌                        | 389 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████▊                        | 399 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████                        | 409 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████▏                       | 419 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████▎                       | 430 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████▌                       | 440 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████▊                       | 450 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████                       | 460 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▏                      | 471 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▎                      | 481 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▌                      | 491 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▊                      | 501 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████                      | 512 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████                      | 522 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▎                     | 532 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▌                     | 542 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▊                     | 552 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████                     | 563 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████                     | 573 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▎                    | 583 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▌                    | 593 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▊                    | 604 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▉                    | 614 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████                    | 624 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▎                   | 634 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▌                   | 645 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▊                   | 655 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▉                   | 665 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████                   | 675 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▎                  | 686 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▌                  | 696 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▊                  | 706 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▉                  | 716 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████                  | 727 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▎                 | 737 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▌                 | 747 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▋                 | 757 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▉                 | 768 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████                 | 778 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▎                | 788 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▌                | 798 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▋                | 808 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▉                | 819 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████                | 829 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▎               | 839 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▍               | 849 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▋               | 860 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▉               | 870 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████               | 880 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▎              | 890 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▍              | 901 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▋              | 911 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▉              | 921 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████              | 931 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▎             | 942 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▍             | 952 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▋             | 962 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▉             | 972 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████             | 983 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▏            | 993 kB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▍            | 1.0 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▋            | 1.0 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▉            | 1.0 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████            | 1.0 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▏           | 1.0 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▍           | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▋           | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▉           | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████           | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▏          | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▍          | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▋          | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▉          | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████          | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▏         | 1.1 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▍         | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▋         | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▉         | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████         | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▏        | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▍        | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▋        | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▊        | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████        | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▏       | 1.2 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▍       | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▋       | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▊       | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████       | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▏      | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▍      | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▌      | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▊      | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████      | 1.3 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▏     | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▍     | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▌     | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▊     | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████     | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▏    | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▍    | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▌    | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▊    | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████    | 1.4 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▏   | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▎   | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▌   | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▊   | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████   | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▏  | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▎  | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▌  | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▊  | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████  | 1.5 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▏ | 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▎ | 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▌ | 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▊ | 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████ | 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████ | 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▎| 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▌| 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▊| 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 1.6 MB 7.6 MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 1.7 MB 7.6 MB/s \n",
      "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
      "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
      "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
      "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
      "  Building wheel for pyLDAvis (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
      "  Building wheel for sklearn (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
     ]
    }
   ],
   "source": [
    "# pyLDAvis 는 scikit-learn 의존성 문제로 해당 실습은 아래 버전으로 진행을 추천\n",
    "!pip install -q scikit-learn==1.0.0\n",
    "!pip install -U -q pyLDAvis==3.4.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1672568395319,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "mI_S8brso1Yr"
   },
   "outputs": [],
   "source": [
    "# ignore warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1kpwaWyzo1Yr"
   },
   "source": [
    "## 라이브러리 로드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 1235,
     "status": "ok",
     "timestamp": 1672568396550,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "7jEmmKL0o1Ys"
   },
   "outputs": [],
   "source": [
    "# 필요 라이브러리를 로드\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3MzjjX3ho1Ys"
   },
   "source": [
    "## 데이터 로드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1151,
     "status": "ok",
     "timestamp": 1672568397696,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "sZd0hR-Vo1Yu",
    "outputId": "6130e778-11d4-448f-c5b0-b1430a0663f5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2645, 5)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"https://bit.ly/seoul-120-text-csv\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "executionInfo": {
     "elapsed": 27,
     "status": "ok",
     "timestamp": 1672568397697,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "qbjUJ2qapEVG",
    "outputId": "90b24cd9-aaf3-4762-a5fd-88441dd72ced"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>번호</th>\n",
       "      <th>분류</th>\n",
       "      <th>제목</th>\n",
       "      <th>내용</th>\n",
       "      <th>내용번호</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2645</td>\n",
       "      <td>복지</td>\n",
       "      <td>아빠 육아휴직 장려금</td>\n",
       "      <td>아빠 육아휴직 장려금   업무개요  남성근로자의 육아휴직을 장려하고 양육에 따른 경...</td>\n",
       "      <td>23522464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2644</td>\n",
       "      <td>경제</td>\n",
       "      <td>[서울산업진흥원] 서울메이드란?</td>\n",
       "      <td>서울산업진흥원 서울메이드란 서울의 감성을 담은 다양하고 새로운 경험을 제공하기 위해...</td>\n",
       "      <td>23194045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2643</td>\n",
       "      <td>환경</td>\n",
       "      <td>(강북구) 정비중</td>\n",
       "      <td>강북구 정비중  업무개요  투명 폐트병을 교환보상하므로 수거율을 높이고 폐기물을 감...</td>\n",
       "      <td>23032485</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     번호  분류                 제목  \\\n",
       "0  2645  복지        아빠 육아휴직 장려금   \n",
       "1  2644  경제  [서울산업진흥원] 서울메이드란?   \n",
       "2  2643  환경          (강북구) 정비중   \n",
       "\n",
       "                                                  내용      내용번호  \n",
       "0  아빠 육아휴직 장려금   업무개요  남성근로자의 육아휴직을 장려하고 양육에 따른 경...  23522464  \n",
       "1  서울산업진흥원 서울메이드란 서울의 감성을 담은 다양하고 새로운 경험을 제공하기 위해...  23194045  \n",
       "2  강북구 정비중  업무개요  투명 폐트병을 교환보상하므로 수거율을 높이고 폐기물을 감...  23032485  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 19,
     "status": "ok",
     "timestamp": 1672568397697,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "9MXU70SjmU4t",
    "outputId": "9a19fdba-fea7-4aa8-8916-1cdc2ed296e9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2645, 5)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 결측치가 있다면 제거\n",
    "df = df.dropna()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 13,
     "status": "ok",
     "timestamp": 1672568397698,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "YXAtNDKco1Yv",
    "outputId": "49ea7be5-5e50-465a-eeef-79457fbdfaa7",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "번호      0\n",
       "분류      0\n",
       "제목      0\n",
       "내용      0\n",
       "내용번호    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 결측치를 확인\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Jw4p_dnIo1Yw"
   },
   "source": [
    "## 문서 만들기\n",
    "* 제목과 내용을 함께 사용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1672568397699,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "UJ9GAlhuo1Yx"
   },
   "outputs": [],
   "source": [
    "df[\"문서\"] = df[\"제목\"] + \" \" + df[\"내용\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lMII7R0go1Yx"
   },
   "source": [
    "## 벡터화\n",
    "\n",
    "* [Bag-of-words model - Wikipedia](https://en.wikipedia.org/wiki/Bag-of-words_model)\n",
    "\n",
    "\n",
    "## CountVectorizer\n",
    "\n",
    "* analyzer : 단어, 문자 단위의 벡터화 방법 정의\n",
    "* ngram_range : BOW 단위 수 (1, 3) 이라면 1개~3개까지 토큰을 묶어서 벡터화\n",
    "* max_df : 어휘를 작성할 때 문서 빈도가 주어진 임계값보다 높은 용어(말뭉치 관련 불용어)는 제외 (기본값=1.0)\n",
    "    * max_df = 0.90 : 문서의 90% 이상에 나타나는 단어 제외\n",
    "    * max_df = 10 : 10개 이상의 문서에 나타나는 단어 제외\n",
    "* min_df : 어휘를 작성할 때 문서 빈도가 주어진 임계값보다 낮은 용어는 제외합니다. 컷오프라고도 합니다.(기본값=1.0)\n",
    "    * min_df = 0.01 : 문서의 1% 미만으로 나타나는 단어 제외\n",
    "    * min_df = 10 : 문서에 10개 미만으로 나타나는 단어 제외\n",
    "* stop_words : 불용어 정의\n",
    "* API Document: https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1672568397699,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "94FBGEqOo1Yy"
   },
   "outputs": [],
   "source": [
    "# 단어들의 출현 빈도(frequency)로 여러 문서들을 벡터화하기 위해 CountVectorizer 사용\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "cv = CountVectorizer(stop_words=[\"돋움\", \"경우\", \"또는\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ft2MvWD57MjT"
   },
   "source": [
    "### 참고: fit, transform, fit_transfrom의 차이점\n",
    "- fit(): 원시 문서에 있는 모든 토큰의 어휘 사전을 배운다\n",
    "- transform(): 문서를 문서 용어 매트릭스로 변환, transform 이후엔 매트릭스로 변환되어 숫자형태로 변경\n",
    "- fit_transform(): 어휘 사전을 배우고 문서 용어 매트릭스를 반환, fit 다음에 변환이 오는 것과 동일하지만 더 효율적으로 구현\n",
    "\n",
    "* API Document: https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer.fit_transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 440,
     "status": "ok",
     "timestamp": 1672568398130,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "YueqXKDbo1Yy"
   },
   "outputs": [],
   "source": [
    "# fit_transform을 사용하여 문장에서 노출되는 feature(특징이 될만한 단어) 수를 합한 변수 Document Term Matrix(이하 dtm)를 생성\n",
    "dtm_cv = cv.fit_transform(df[\"문서\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1672568398131,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "Rr_Gj0Bro1Yy"
   },
   "outputs": [],
   "source": [
    "# cv.vocabulary_ 를 봅니다.\n",
    "# cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1672568398131,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "5dRGKkOamU4w"
   },
   "outputs": [],
   "source": [
    "cv_cols = cv.get_feature_names_out()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1440,
     "status": "ok",
     "timestamp": 1672568399567,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "S6sZ_cIdo1Yz",
    "outputId": "1bb72030-718b-4bb6-9c1a-c7aab615e9a0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "03월       1\n",
       "용액속에      1\n",
       "용액을       1\n",
       "용액이       1\n",
       "용어        1\n",
       "       ... \n",
       "대한      394\n",
       "서울시     578\n",
       "어떻게     597\n",
       "있습니다    685\n",
       "있는      718\n",
       "Length: 56651, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 각 row에서 전체 단어가방에 있는 어휘에서 등장하는 단어에 대한 one-hot-vector를 확인\n",
    "# toarray()로 희소 행렬(sparse matrix, 행렬의 값이 대부분 '0'인 행렬)을 NumPy array 배열로 변환하여 값을 확인\n",
    "\n",
    "pd.DataFrame(dtm_cv.toarray(), columns=cv_cols).sum().sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tZKMAZ5go1Yz"
   },
   "source": [
    "## 잠재 디리클레 할당(Latent Dirichlet Allocation, LDA)\n",
    "\n",
    "* API documentation: https://pyldavis.readthedocs.io/en/latest/modules/API.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1672568399567,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "Tj6AiX1Po1Y0",
    "outputId": "b29b3892-3ec5-407f-b34a-1796f1de7a46"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "행정        1098\n",
       "경제         823\n",
       "복지         217\n",
       "환경         124\n",
       "주택도시계획     110\n",
       "문화관광        96\n",
       "교통          90\n",
       "안전          51\n",
       "건강          23\n",
       "여성가족        13\n",
       "Name: 분류, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 정답인 '분류'의 유일한 값을 확인하여 주제 수를 확인\n",
    "df[\"분류\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1672568399567,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "FQBuD_fJo1Y0"
   },
   "outputs": [],
   "source": [
    "# 주어진 문서에 대하여 각 문서에 어떤 주제들이 존재하는지를 확인하는 잠재 디리클레 분석(LDA)을 불러옴\n",
    "# n_components에 넣을 하이퍼파라미터 NUM_TOPICS로 주제수를 설정(기본값=10)\n",
    "# max_iter는 훈련 데이터(epoch라고도 함)에 대한 최대 패스 수(기본값=10)\n",
    "\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "NUM_TOPICS = 10\n",
    "LDA_model = LatentDirichletAllocation(n_components=NUM_TOPICS, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 12616,
     "status": "ok",
     "timestamp": 1672568412181,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "WGUq-nhkmU4x",
    "outputId": "6cf54776-8241-4a4f-946f-9512f5a01926"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LatentDirichletAllocation(random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LatentDirichletAllocation</label><div class=\"sk-toggleable__content\"><pre>LatentDirichletAllocation(random_state=42)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LatentDirichletAllocation(random_state=42)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LDA_model 에 dtm_cv 를 넣어 학습\n",
    "LDA_model.fit(dtm_cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BxtYGPLMmU4x"
   },
   "source": [
    "### pyLDAvis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "executionInfo": {
     "elapsed": 31,
     "status": "ok",
     "timestamp": 1672568412181,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "I5bR1zgTmU4y"
   },
   "outputs": [],
   "source": [
    "# !pip install -U -q pyLDAvis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 954
    },
    "executionInfo": {
     "elapsed": 10159,
     "status": "ok",
     "timestamp": 1672568422312,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "Sbyb39Szo1Y0",
    "outputId": "6987282c-06b6-438f-8f88-b76a9cd89c13"
   },
   "outputs": [],
   "source": [
    "# 토픽 모델링에 이용되는 LDA 모델의 학습 결과를 시각화하는 Python 라이브러리인 pyLDAvis를 불러옴\n",
    "# mds(Multi-Dimensional Scaling)는 데이터 포인트 간의 거리를 보존하면서 차원을 축소하는 기법\n",
    "# t-SNE(t-Stochastic Neighbor Embedding)은 고차원 데이터를 특히 2, 3차원 등으로 줄여 가시화하는데에 유용\n",
    "\n",
    "import pyLDAvis.sklearn\n",
    "\n",
    "pyLDAvis.enable_notebook()\n",
    "pyLDAvis.sklearn.prepare(LDA_model, dtm_cv, cv, mds='tsne')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bduBjLQZo1Y1"
   },
   "source": [
    "## TF-IDF(Term Frequency - Inverse Document Frequency)\n",
    "\n",
    "## TfidfVectorizer\n",
    "\n",
    "TF-IDF 인코딩은 단어를 갯수 그대로 카운트하지 않고 모든 문서에 공통적으로 들어있는 단어(낮은 구별력)의 경우 가중치를 축소하는 방법\n",
    "\n",
    "매개변수\n",
    "* norm='l2' 각 문서의 피처 벡터를 어떻게 벡터 정규화 할지 정한다. \n",
    "    - L2 : 벡터의 각 원소의 제곱의 합이 1이 되도록 만드는 것이고 기본 값\n",
    "    - L1 : 벡터의 각 원소의 절댓값의 합이 1이 되도록 크기를 조절\n",
    "* smooth_idf=False\n",
    "    - 피처를 만들 때 0으로 나오는 항목에 대해 작은 값을 더해서(스무딩을 해서) 피처를 만들지 아니면 그냥 생성할지를 결정\n",
    "* sublinear_tf=False\n",
    "* use_idf=True\n",
    "    - TF-IDF를 사용해 피처를 만들 것인지 아니면 단어 빈도 자체를 사용할 것인지 여부\n",
    "* API Document: https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html#sklearn.feature_extraction.text.TfidfVectorizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1672568422312,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "tEUNepLOo1Y1",
    "outputId": "17b8beba-8852-42e6-f731-93ec0fd7e43f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(stop_words=['돋움', '경우', '또는', '있습니다', '있는', '합니다'])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TF-IDF 방식으로 단어의 가중치를 조정한 BOW 인코딩하여 벡터화하기 위해 TfidfVectorizer를 사용\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tfidf = TfidfVectorizer(stop_words=[\"돋움\", \"경우\", \"또는\", \"있습니다\", \"있는\", \"합니다\"])\n",
    "tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "executionInfo": {
     "elapsed": 483,
     "status": "ok",
     "timestamp": 1672568422781,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "UKEXeuKQo1Y1"
   },
   "outputs": [],
   "source": [
    "# 문장에서 노출되는 feature(특징이 될만한 단어) 수를 합한 변수 Document Term Matrix(이하 dtm)를 생성\n",
    "dtm_tfidf = tfidf.fit_transform(df[\"문서\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1672568422782,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "XLkWfsggo1Y1"
   },
   "outputs": [],
   "source": [
    "# tfidf.vocabulary_\n",
    "cols_tfidf = tfidf.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 363
    },
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1672568422782,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "n8W95ppMo1Y2",
    "outputId": "8a8eea4c-3d20-40e3-d948-41692395ab02"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-faf48d40-3f7a-42df-b5ec-f0237f49136f\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>의한</th>\n",
       "      <td>15.021840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>무엇입니까</th>\n",
       "      <td>15.270257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>이상</th>\n",
       "      <td>15.577954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>관한</th>\n",
       "      <td>16.593598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>무엇인가요</th>\n",
       "      <td>16.650743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>따라</th>\n",
       "      <td>16.652594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>대한</th>\n",
       "      <td>18.866037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>있나요</th>\n",
       "      <td>19.707343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>서울시</th>\n",
       "      <td>22.586695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>어떻게</th>\n",
       "      <td>37.924574</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-faf48d40-3f7a-42df-b5ec-f0237f49136f')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-faf48d40-3f7a-42df-b5ec-f0237f49136f button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-faf48d40-3f7a-42df-b5ec-f0237f49136f');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "               0\n",
       "의한     15.021840\n",
       "무엇입니까  15.270257\n",
       "이상     15.577954\n",
       "관한     16.593598\n",
       "무엇인가요  16.650743\n",
       "따라     16.652594\n",
       "대한     18.866037\n",
       "있나요    19.707343\n",
       "서울시    22.586695\n",
       "어떻게    37.924574"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dtm_tf를 axis=0(수직 방향으로) 기준으로 합계를 낸 dist 변수를 생성\n",
    "# dist 변수를 vocabulary_ 순으로 정렬하여 비율을 확인\n",
    "dist = np.sum(dtm_tfidf, axis=0)\n",
    "pd.DataFrame(dist, columns=cols_tfidf).T.sort_values(by=0).tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 485
    },
    "executionInfo": {
     "elapsed": 459,
     "status": "ok",
     "timestamp": 1672568423238,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "l61luP9mo1Y2",
    "outputId": "958dd92d-c345-4223-c647-4ec19616edff"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-6c86b00c-8457-4116-878a-f109187073d8\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>03월</th>\n",
       "      <th>08년</th>\n",
       "      <th>10</th>\n",
       "      <th>100명이상인</th>\n",
       "      <th>100세가</th>\n",
       "      <th>10만원</th>\n",
       "      <th>10만원상당</th>\n",
       "      <th>10명이고</th>\n",
       "      <th>10인승</th>\n",
       "      <th>10인의</th>\n",
       "      <th>...</th>\n",
       "      <th>힐링프로그램을</th>\n",
       "      <th>힐링하는</th>\n",
       "      <th>힐스테이트</th>\n",
       "      <th>힘들</th>\n",
       "      <th>힘들경우</th>\n",
       "      <th>힘들고</th>\n",
       "      <th>힘쓰고있습니다</th>\n",
       "      <th>힘쓴다</th>\n",
       "      <th>힘을</th>\n",
       "      <th>힘이</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2640</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2641</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2642</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2643</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2644</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2645 rows × 56648 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6c86b00c-8457-4116-878a-f109187073d8')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-6c86b00c-8457-4116-878a-f109187073d8 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-6c86b00c-8457-4116-878a-f109187073d8');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "      03월  08년   10  100명이상인  100세가  10만원  10만원상당  10명이고  10인승  10인의  ...  \\\n",
       "0     0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "1     0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "2     0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "3     0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "4     0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "...   ...  ...  ...      ...    ...   ...     ...    ...   ...   ...  ...   \n",
       "2640  0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "2641  0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "2642  0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "2643  0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "2644  0.0  0.0  0.0      0.0    0.0   0.0     0.0    0.0   0.0   0.0  ...   \n",
       "\n",
       "      힐링프로그램을  힐링하는  힐스테이트   힘들  힘들경우  힘들고  힘쓰고있습니다  힘쓴다   힘을   힘이  \n",
       "0         0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "1         0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "2         0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "3         0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "4         0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "...       ...   ...    ...  ...   ...  ...      ...  ...  ...  ...  \n",
       "2640      0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "2641      0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "2642      0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "2643      0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "2644      0.0   0.0    0.0  0.0   0.0  0.0      0.0  0.0  0.0  0.0  \n",
       "\n",
       "[2645 rows x 56648 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 각 row에서 전체 단어가방에 있는 어휘에서 등장하는 단어에 대한 가중치를 적용한 vector를 확인\n",
    "# toarray()로 희소 행렬(sparse matrix, 행렬의 값이 대부분 '0'인 행렬)을 NumPy array 배열로 변환하여 값을 확인\n",
    "pd.DataFrame(dtm_tfidf.toarray(), columns=cols_tfidf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5SfyYdvZo1Y3"
   },
   "source": [
    "## 코사인 유사도\n",
    "* API Document: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.pairwise.cosine_similarity.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1672568423239,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "AC98n4w5o1Y3"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "similarity_simple_pair = cosine_similarity(dtm_tfidf[0] , dtm_tfidf)\n",
    "result_list = similarity_simple_pair.tolist()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 363
    },
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1672568423239,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "FJFB-n4Lo1Y3",
    "outputId": "9104f08d-31a5-4f67-87ce-d845b7f965ea"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-055d277a-88c2-4f1b-9f58-22d6f3470f13\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>분류</th>\n",
       "      <th>제목</th>\n",
       "      <th>유사도</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>복지</td>\n",
       "      <td>아빠 육아휴직 장려금</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1772</th>\n",
       "      <td>경제</td>\n",
       "      <td>도시계획시설부지 재결신청 이후 진행단계는 어떤 과정을 거칩니까?</td>\n",
       "      <td>0.057158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>850</th>\n",
       "      <td>경제</td>\n",
       "      <td>주민대표회의 구성원 몇명입니까?</td>\n",
       "      <td>0.056152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>539</th>\n",
       "      <td>행정</td>\n",
       "      <td>행려자도 아니고 시설수용자도 아닌 사람이 살고 있던 비닐하우스에서 화상을 입었습니다...</td>\n",
       "      <td>0.051952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>복지</td>\n",
       "      <td>광진맘택시 운영(임산부,영아 양육가정 전용 택시)</td>\n",
       "      <td>0.048404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>경제</td>\n",
       "      <td>[농업기술센터] 후계농업경영인 선정 및 청년창업형 후계농업경영인 신청 안내</td>\n",
       "      <td>0.046280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>행정</td>\n",
       "      <td>[시ㆍ구정외 타기관 관련 상담] 고용노동부 [일자리 안정자금]</td>\n",
       "      <td>0.046037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>경제</td>\n",
       "      <td>[농업기술센터] 도시농업전문가양성교육 신청</td>\n",
       "      <td>0.043873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>행정</td>\n",
       "      <td>찾아가는 아버지교실</td>\n",
       "      <td>0.043034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>경제</td>\n",
       "      <td>[농업기술센터] 귀농창업 평일반 교육 신청</td>\n",
       "      <td>0.041945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-055d277a-88c2-4f1b-9f58-22d6f3470f13')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-055d277a-88c2-4f1b-9f58-22d6f3470f13 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-055d277a-88c2-4f1b-9f58-22d6f3470f13');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "      분류                                                 제목       유사도\n",
       "0     복지                                        아빠 육아휴직 장려금  1.000000\n",
       "1772  경제                도시계획시설부지 재결신청 이후 진행단계는 어떤 과정을 거칩니까?  0.057158\n",
       "850   경제                                  주민대표회의 구성원 몇명입니까?  0.056152\n",
       "539   행정  행려자도 아니고 시설수용자도 아닌 사람이 살고 있던 비닐하우스에서 화상을 입었습니다...  0.051952\n",
       "3     복지                        광진맘택시 운영(임산부,영아 양육가정 전용 택시)  0.048404\n",
       "155   경제          [농업기술센터] 후계농업경영인 선정 및 청년창업형 후계농업경영인 신청 안내  0.046280\n",
       "35    행정                 [시ㆍ구정외 타기관 관련 상담] 고용노동부 [일자리 안정자금]  0.046037\n",
       "141   경제                            [농업기술센터] 도시농업전문가양성교육 신청  0.043873\n",
       "174   행정                                         찾아가는 아버지교실  0.043034\n",
       "233   경제                            [농업기술센터] 귀농창업 평일반 교육 신청  0.041945"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"유사도\"] = result_list\n",
    "df[[\"분류\", \"제목\", \"유사도\"]].sort_values(by=\"유사도\", ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1672568423240,
     "user": {
      "displayName": "송영숙",
      "userId": "01138230862948206591"
     },
     "user_tz": -540
    },
    "id": "M7dYarIhmU40"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "256px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}