sample/nlp-skip_thoughts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Ingestion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def get_raw_data():\n",
    "    # Get file from http://archive.ics.uci.edu/ml/datasets/News+Aggregator\n",
    "    df = pd.read_csv(\n",
    "        \"./../data/demo_data/NewsAggregatorDataset/newsCorpora.csv\", \n",
    "        sep='\\t', \n",
    "        names=['id', 'headline', 'url', 'publisher', 'category', 'story', 'hostname', 'timestamp']\n",
    "    )\n",
    "    \n",
    "    # Category: b = business, t = science and technology, e = entertainment, m = health\n",
    "    return df[['category', 'headline']]\n",
    "\n",
    "df = get_raw_data()\n",
    "df.head()\n",
    "\n",
    "train_df, test_df = train_test_split(df, test_size=0.2)\n",
    "\n",
    "x_train = train_df['headline'].values\n",
    "y_train = train_df['category'].values\n",
    "x_test = test_df['headline'].values\n",
    "y_test = test_df['category'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = [sentence[:300] for sentence in x_train]\n",
    "x_test = [sentence[:300] for sentence in x_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'e': 0, 't': 1, 'm': 2, 'b': 3}\n",
      "{0: 'e', 1: 't', 2: 'm', 3: 'b'}\n"
     ]
    }
   ],
   "source": [
    "def build_y_encoder(y):\n",
    "    label2idx = {}\n",
    "    idx2label = {}\n",
    "    for label in list(set(y)):\n",
    "        label2idx[label] = len(label2idx)\n",
    "        idx2label[label2idx[label]] = label\n",
    "        \n",
    "    print(label2idx)\n",
    "    print(idx2label)\n",
    "    \n",
    "    return label2idx, idx2label\n",
    "    \n",
    "label2idx, idx2label = build_y_encoder(y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Skip-Thoughts Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found dictionary.txt in ./skip_thoughts/\n",
      "Found utable.npy in ./skip_thoughts/\n",
      "Found btable.npy in ./skip_thoughts/\n",
      "Found uni_skip.npz in ./skip_thoughts/\n",
      "Found bi_skip.npz in ./skip_thoughts/\n",
      "Found uni_skip.npz.pkl in ./skip_thoughts/\n",
      "Found bi_skip.npz.pkl in ./skip_thoughts/\n",
      "2018-09-29 15:52:09.704205 build\n"
     ]
    }
   ],
   "source": [
    "# Copyright 2018 Edward Ma. All Rights Reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "# ==============================================================================\n",
    "\n",
    "import numpy as np\n",
    "import os, datetime\n",
    "from torch import LongTensor\n",
    "from torch.autograd import Variable\n",
    "from skipthoughts import UniSkip, BiSkip\n",
    "\n",
    "class SkipThoughtsEmbeddings:\n",
    "    DICTIONARY_URL = \"http://www.cs.toronto.edu/~rkiros/models/dictionary.txt\"\n",
    "    UNISKIP_URL = \"http://www.cs.toronto.edu/~rkiros/models/utable.npy\"\n",
    "    BISKIP_URL = \"http://www.cs.toronto.edu/~rkiros/models/btable.npy\"\n",
    "    UNISKIPS_URL = \"http://www.cs.toronto.edu/~rkiros/models/uni_skip.npz\"\n",
    "    BISKIPS_URL = \"http://www.cs.toronto.edu/~rkiros/models/bi_skip.npz\"\n",
    "    UNISKIPS_PKL_URL = \"http://www.cs.toronto.edu/~rkiros/models/uni_skip.npz.pkl\"\n",
    "    BISKIPS_PKL_URL = \"http://www.cs.toronto.edu/~rkiros/models/bi_skip.npz.pkl\"\n",
    "    \n",
    "    def __init__(self, model_dir, algorithm='uniskip', tokenizer=None, verbose=0):\n",
    "        self.verbose = verbose\n",
    "        \n",
    "        self.model_dir = model_dir\n",
    "        self.algorithm = algorithm\n",
    "        self.vocab = {}\n",
    "        self.vocabs = []\n",
    "        if tokenizer is None:\n",
    "            self.tokenizer = self._tokenizer_space\n",
    "        else:\n",
    "            self.tokenizer = tokenizer\n",
    "        self.max_sentence_len = -1\n",
    "        \n",
    "    def _tokenizer_space(self, sentence):\n",
    "        return sentence.split(' ')\n",
    "        \n",
    "    def _download(self, src, dest_dir, dest_file, unzip):\n",
    "        if not os.path.exists(dest_dir):\n",
    "            os.makedirs(dest_dir)\n",
    "    \n",
    "        if dest_file is None:\n",
    "            dest_file = os.path.basename(src)\n",
    "            \n",
    "        if not os.path.exists(dest_dir + dest_file):\n",
    "            print('Downloading from %s' % (src))\n",
    "            file = urllib.request.urlopen(url)\n",
    "            with open(dest_dir + dest_file,'wb') as output:\n",
    "                output.write(file.read())\n",
    "        else:\n",
    "            print('Found %s in %s' % (dest_file, dest_dir))\n",
    "        \n",
    "    def downloads(self, dest_dir, sources=None):\n",
    "        if sources is None:\n",
    "            sources = [self.DICTIONARY_URL, self.UNISKIP_URL, self.BISKIP_URL, \n",
    "                       self.UNISKIPS_URL, self.BISKIPS_URL, self.UNISKIPS_PKL_URL, \n",
    "                       self.BISKIPS_PKL_URL]\n",
    "        \n",
    "        for src in sources:\n",
    "            self._download(src=src, dest_dir=dest_dir, dest_file=None, unzip=False)\n",
    "        \n",
    "    def build_vocab(self, sentences, clear_vocab=True, max_sentence_len=-1):\n",
    "        if clear_vocab:\n",
    "            self.vocab = {}\n",
    "            \n",
    "        self.max_sentence_len = max_sentence_len\n",
    "        \n",
    "        for sentence in sentences:\n",
    "            words = self.tokenizer(sentence)\n",
    "            if max_sentence_len == -1:\n",
    "                self.max_sentence_len = max(self.max_sentence_len, len(words))\n",
    "\n",
    "            for word in words:\n",
    "                if word not in self.vocab:\n",
    "                    self.vocabs.append(word)\n",
    "                    # Reserve the first one for padding\n",
    "                    self.vocab[word] = len(self.vocab) + 1\n",
    "\n",
    "    def process(self, sentences):\n",
    "        word_id_sentences = []\n",
    "        for sentence in sentences:\n",
    "            word_ids = [self.vocab[w] for w in self.tokenizer(sentence) if w in self.vocab]\n",
    "            \n",
    "            if self.max_sentence_len > len(word_ids):\n",
    "                for i in range(0, self.max_sentence_len-len(word_ids)):\n",
    "                    word_ids.append(0)\n",
    "            elif self.max_sentence_len < len(word_ids):\n",
    "                word_ids = word_ids[:self.max_sentence_len]\n",
    "                    \n",
    "            word_id_sentences.append(word_ids)\n",
    "            \n",
    "        return word_id_sentences\n",
    "    \n",
    "    def get_algorithm(self, words, model_dir=None):\n",
    "        if model_dir is None:\n",
    "            model_dir = self.model_dir\n",
    "            \n",
    "        if self.algorithm == 'uniskip':\n",
    "            return UniSkip(model_dir, words)\n",
    "        else:\n",
    "            return BiSkip(model_dir, words)\n",
    "        \n",
    "    def _convert_to_numpy_layer(self, layer):\n",
    "        return layer.detach().numpy()        \n",
    "\n",
    "    def predict(self, sentences, output_format='torch'):\n",
    "        transformed_sentences = self.process(sentences)\n",
    "        \n",
    "        algo = self.get_algorithm(self.vocabs)\n",
    "        inputs = Variable(LongTensor(transformed_sentences))\n",
    "        outpus = algo(inputs, lengths=[len(words) for words in transformed_sentences])\n",
    "        \n",
    "        if output_format == 'np':\n",
    "            return self._convert_to_numpy_layer(outpus)\n",
    "        elif output_format == 'torch':\n",
    "            return outpus\n",
    "        \n",
    "    def predict_batch(self, sentences, output_format='torch', batch_size=1000):\n",
    "        batches = [sentences[i * batch_size:(i + 1) * batch_size] for i in range((len(sentences) + batch_size-1) // batch_size)]\n",
    "\n",
    "        results = []\n",
    "        for batch in batches:\n",
    "            results.append(skip_thoughts_emb.predict(sentences=batch, output_format=output_format))\n",
    "\n",
    "        if output_format == 'np':\n",
    "            return np.concatenate(results, axis=0)\n",
    "        elif output_format == 'torch':\n",
    "            return torch.cat(results, 0)\n",
    "\n",
    "dest_dir = './skip_thoughts/'\n",
    "skip_thoughts_emb = SkipThoughtsEmbeddings(model_dir=dest_dir)\n",
    "skip_thoughts_emb.downloads(dest_dir=dest_dir)\n",
    "print(datetime.datetime.now(), 'build')\n",
    "skip_thoughts_emb.build_vocab(x_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-09-29 15:52:11.043806 get\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.25 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "Warning: 96789/930911 words are not in dictionary, thus set UNK\n",
      "2018-09-29 17:14:50.400833 done\n"
     ]
    }
   ],
   "source": [
    "print(datetime.datetime.now(), 'get')\n",
    "x_train_t = skip_thoughts_emb.predict_batch(x_train, output_format='np')\n",
    "x_test_t = skip_thoughts_emb.predict_batch(x_test, output_format='np')\n",
    "print(datetime.datetime.now(), 'done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/scipy/optimize/linesearch.py:313: LineSearchWarning: The line search algorithm did not converge\n",
      "  warn('The line search algorithm did not converge', LineSearchWarning)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/utils/optimize.py:195: UserWarning: Line Search failed\n",
      "  warnings.warn('Line Search failed')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='newton-cg', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LogisticRegression(solver='newton-cg')\n",
    "model.fit(x_train_t, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict(x_test_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.80643672174612946"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(y_test, y_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}