02 - TF-IDF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "import json\n",
    "import gzip\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = []\n",
    "with zipfile.ZipFile('kompas.zip') as f:\n",
    "    for file_ in f.namelist():\n",
    "        if file_.endswith('.txt'):\n",
    "            with f.open(file_, 'r') as tf:\n",
    "                docs.append(tf.read().decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with zipfile.ZipFile('tempo.zip') as f:\n",
    "    for file_ in f.namelist():\n",
    "        if file_.endswith('.txt'):\n",
    "            with f.open(file_, 'r') as tf:\n",
    "                docs.append(tf.read().decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28394"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('word2idx.json', 'r') as f:\n",
    "    w2i = json.load(f)\n",
    "    \n",
    "with gzip.open('idx2word.txt.gz', 'rt', encoding='utf-8') as f:\n",
    "    i2w = [line.strip() for line in f]\n",
    "    \n",
    "vec = np.load('vec.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer(\n",
    "    input='content',\n",
    "    analyzer='word',\n",
    "    lowercase=True,\n",
    "    vocabulary=i2w,\n",
    "    dtype=np.float32\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tfidf_vectorizer.fit_transform(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((28394, 1298176), scipy.sparse.csr.csr_matrix)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, type(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['</s>', 'orang', 'memiliki', 'indonesia', 'salah']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_vectorizer.get_feature_names()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((28394, 1298176), (1298176, 300))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, vec.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_vec = X.dot(vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28394, 300)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_vec.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x1298176 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 189 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malaysia 0.43007773\n",
      "desa 0.41204834\n",
      "tenggulun 0.3324931\n",
      "gufron 0.25848386\n",
      "maskun 0.2327798\n",
      "ketertinggalan 0.19182579\n",
      "warga 0.15146072\n",
      "amrozi 0.14649215\n",
      "terpencil 0.1312966\n",
      "pergi 0.11545662\n",
      "modernitas 0.10564377\n",
      "pesantren 0.10369952\n",
      "mengaspal 0.09870996\n",
      "ladang 0.09766841\n",
      "keterpencilan 0.09647778\n",
      "pondok 0.094817035\n",
      "tertinggal 0.089977376\n",
      "panen 0.08944007\n",
      "keluarga 0.08820153\n",
      "ringgit 0.07673032\n",
      "kerja 0.076223925\n",
      "bayangan 0.07427827\n",
      "jiran 0.07249075\n",
      "lamongan 0.069936424\n",
      "kemarau 0.06671247\n",
      "kilometer 0.06669638\n",
      "jalan 0.063428506\n",
      "pulang 0.06315721\n",
      "penduduk 0.05990078\n",
      "simbol 0.05792086\n",
      "kali 0.054614376\n",
      "tinggal 0.05347323\n",
      "slumun 0.052821886\n",
      "sluman 0.052821886\n",
      "diangan 0.052821886\n",
      "kesan 0.052793946\n",
      "terkucilkan 0.05079388\n",
      "identity 0.05079388\n",
      "nama 0.04949481\n",
      "pembenarannya 0.04935498\n",
      "arqam 0.04935498\n",
      "ampang 0.04935498\n",
      "ditenteng 0.047326975\n",
      "mengubah 0.046860643\n",
      "tasik 0.04655596\n",
      "tadah 0.04655596\n",
      "bangunan 0.046195768\n",
      "khozin 0.044771988\n",
      "sodik 0.044295277\n",
      "berdakwah 0.044295277\n",
      "berubah 0.04425664\n",
      "sekolah 0.043905985\n",
      "terasing 0.043860074\n",
      "namanya 0.043699298\n",
      "dadapan 0.043459725\n",
      "solokuro 0.042743973\n",
      "persyaratannya 0.042743973\n",
      "belenggu 0.042743973\n",
      "perginya 0.041561637\n",
      "mengaku 0.041328836\n",
      "rumah 0.041240525\n",
      "dibangunnya 0.04082837\n",
      "hamparan 0.04018899\n",
      "mudahnya 0.03999282\n",
      "card 0.039804053\n",
      "mengering 0.03944664\n",
      "mudah 0.03934104\n",
      "hiasan 0.039277073\n",
      "sirna 0.038800362\n",
      "johor 0.03865105\n",
      "ranting 0.03850606\n",
      "dasar 0.03795331\n",
      "red 0.03786159\n",
      "pepohonan 0.037476454\n",
      "kandungnya 0.037139136\n",
      "handphone 0.03703157\n",
      "kedelai 0.036430646\n",
      "kewarganegaraan 0.036155254\n",
      "sekolahnya 0.03548737\n",
      "kemakmuran 0.035409827\n",
      "enam 0.035223465\n",
      "menengok 0.034371275\n",
      "bertemu 0.033857062\n",
      "diperketat 0.033672232\n",
      "jagung 0.03361816\n",
      "jaraknya 0.03345936\n",
      "bondong 0.033356234\n",
      "hitung 0.033255182\n",
      "berbondong 0.033205416\n",
      "melekat 0.033205416\n",
      "santri 0.03301115\n",
      "dimilikinya 0.03255614\n",
      "terkumpul 0.032512832\n",
      "mengaitkan 0.032099236\n",
      "juta 0.03185072\n",
      "putrinya 0.031791344\n",
      "mengamati 0.031791344\n",
      "bangga 0.03171724\n",
      "padi 0.031260632\n",
      "makmur 0.031160926\n",
      "tani 0.031160926\n",
      "mustahil 0.030873211\n",
      "populer 0.030399324\n",
      "buronan 0.029838549\n",
      "kering 0.029838549\n",
      "tempatnya 0.029664775\n",
      "remaja 0.029616216\n",
      "lumpur 0.029357249\n",
      "melintasi 0.029243825\n",
      "ketemu 0.029199166\n",
      "puasa 0.02913292\n",
      "dana 0.029047525\n",
      "kuala 0.028876366\n",
      "mengandalkan 0.028712362\n",
      "diatasi 0.028592795\n",
      "memiliki 0.028373295\n",
      "kenal 0.028195478\n",
      "dihitung 0.028123248\n",
      "sepekan 0.027500387\n",
      "hasil 0.027409028\n",
      "kartu 0.027193438\n",
      "cari 0.02698926\n",
      "temannya 0.026684353\n",
      "masuknya 0.026435202\n",
      "perumahan 0.02624691\n",
      "kemajuan 0.026077345\n",
      "dibenarkan 0.025878886\n",
      "investigasi 0.025754545\n",
      "negeri 0.02544205\n",
      "lantaran 0.025399055\n",
      "saudara 0.02509644\n",
      "terdengar 0.024932751\n",
      "setahun 0.024820352\n",
      "tua 0.024514945\n",
      "berdiri 0.02431007\n",
      "lepas 0.024276728\n",
      "pertanian 0.024268424\n",
      "anggotanya 0.023839444\n",
      "contoh 0.023711674\n",
      "hujan 0.023638008\n",
      "televisi 0.02360158\n",
      "dimiliki 0.023508089\n",
      "nasib 0.02347259\n",
      "kepala 0.023253156\n",
      "peledakan 0.023244413\n",
      "kampung 0.023117613\n",
      "teman 0.023097884\n",
      "hasilnya 0.023000382\n",
      "drs 0.022923728\n",
      "penangkapan 0.02259261\n",
      "bagus 0.022568934\n",
      "istri 0.022510229\n",
      "pemuda 0.022486938\n",
      "lantai 0.021968678\n",
      "berkembang 0.021849649\n",
      "tercatat 0.021813976\n",
      "yayasan 0.02175341\n",
      "papar 0.02161\n",
      "luas 0.021561487\n",
      "putra 0.021229941\n",
      "duduk 0.021078633\n",
      "membangun 0.021052403\n",
      "jiwa 0.020872528\n",
      "penuh 0.020296987\n",
      "delapan 0.020278275\n",
      "memasuki 0.020259634\n",
      "alam 0.019828463\n",
      "bom 0.01966391\n",
      "data 0.019657306\n",
      "total 0.019242765\n",
      "menimbulkan 0.019224577\n",
      "islam 0.019110898\n",
      "meninggalkan 0.018882522\n",
      "muncul 0.018809587\n",
      "kecamatan 0.018737698\n",
      "kabupaten 0.017691266\n",
      "mencari 0.017531287\n",
      "berhasil 0.01686961\n",
      "kawasan 0.016676856\n",
      "bantuan 0.016510565\n",
      "jawa 0.016188754\n",
      "persen 0.015895724\n",
      "kondisi 0.014955636\n",
      "timur 0.014387328\n",
      "tim 0.0135202855\n",
      "salah 0.012297584\n",
      "anggota 0.011606086\n",
      "negara 0.010836016\n",
      "ketua 0.010392525\n"
     ]
    }
   ],
   "source": [
    "row, col = X[0, :].nonzero()\n",
    "\n",
    "example = []\n",
    "for i in col:\n",
    "    example.append((i2w[i], X[0, i]))\n",
    "\n",
    "for w, val in sorted(example, key=lambda x: x[1], reverse=True):\n",
    "    print(w, val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('doc_vec.npy', doc_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}