name_matching-word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn import svm, linear_model\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import roc_curve, auc,f1_score\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from collections import defaultdict\n",
    "import unicodedata\n",
    "import random\n",
    "random.seed(37)\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from annoy import AnnoyIndex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/home/akash/projects/data_analysis/company_name_matching/data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = pd.read_csv(path+\"G.csv\", delimiter=\"|\")\n",
    "train = pd.read_csv(path+\"STrain.csv\", delimiter=\"|\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>634022</td>\n",
       "      <td>PRIMCOM SA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>324497</td>\n",
       "      <td>The David Isaacs Fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>280848</td>\n",
       "      <td>Bramor Enterprises Limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>432662</td>\n",
       "      <td>NAVEXIM S.A.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>524224</td>\n",
       "      <td>Magal Group SA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company_id                        name\n",
       "0  634022      PRIMCOM SA                \n",
       "1  324497      The David Isaacs Fund     \n",
       "2  280848      Bramor Enterprises Limited\n",
       "3  432662      NAVEXIM S.A.              \n",
       "4  524224      Magal Group SA            "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 450256 entries, 0 to 450255\n",
      "Data columns (total 2 columns):\n",
      "company_id    450256 non-null int64\n",
      "name          450256 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 6.9+ MB\n"
     ]
    }
   ],
   "source": [
    "companies.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>TRATTAMENTO Ltd RIFIUTI METROPOLITANI SPA  SIGLABILE TRM SPA</td>\n",
       "      <td>177358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A IRL Fuund</td>\n",
       "      <td>568472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>BMR-500 Kendall LLC 1 Mezz GmbH</td>\n",
       "      <td>195692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Solich GmbH KG</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Drzyzzga Funds Logi. sp. z oo</td>\n",
       "      <td>404178</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_index                                                          name  \\\n",
       "0  0            TRATTAMENTO Ltd RIFIUTI METROPOLITANI SPA  SIGLABILE TRM SPA   \n",
       "1  1            A IRL Fuund                                                    \n",
       "2  2            BMR-500 Kendall LLC 1 Mezz GmbH                                \n",
       "3  3            Solich GmbH KG                                                 \n",
       "4  4            Drzyzzga Funds Logi. sp. z oo                                  \n",
       "\n",
       "   company_id  \n",
       "0  177358      \n",
       "1  568472      \n",
       "2  195692      \n",
       "3 -1           \n",
       "4  404178      "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100000 entries, 0 to 99999\n",
      "Data columns (total 3 columns):\n",
      "train_index    100000 non-null int64\n",
      "name           100000 non-null object\n",
      "company_id     100000 non-null int64\n",
      "dtypes: int64(2), object(1)\n",
      "memory usage: 2.3+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_index</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>THEking'S ROYAL HUSSARS OFFI. TRUST' TRUST</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Southern Powe rcompany SICAV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>BMO S&amp;P/TSX Ladde. Share ETF Index</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>PaI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Clearview Two</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   test_index                                        name\n",
       "0  0           THEking'S ROYAL HUSSARS OFFI. TRUST' TRUST\n",
       "1  1           Southern Powe rcompany SICAV              \n",
       "2  2           BMO S&P/TSX Ladde. Share ETF Index        \n",
       "3  3           PaI                                       \n",
       "4  4           Clearview Two                             "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv(path+\"STest.csv\", delimiter=\",\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100000 entries, 0 to 99999\n",
      "Data columns (total 2 columns):\n",
      "test_index    100000 non-null int64\n",
      "name          99999 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 1.5+ MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u'train_index', u'name', u'company_id'], dtype='object')\n",
      "Index([u'test_index', u'name'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print train.columns\n",
    "print test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 3)\n",
      "99365\n"
     ]
    }
   ],
   "source": [
    "print train.shape\n",
    "print train[\"name\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 2)\n",
      "99323\n"
     ]
    }
   ],
   "source": [
    "print test.shape\n",
    "print test[\"name\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1075\n"
     ]
    }
   ],
   "source": [
    "print len(set(test[\"name\"]) & set(train[\"name\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>634022</td>\n",
       "      <td>PRIMCOM SA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>324497</td>\n",
       "      <td>The David Isaacs Fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>280848</td>\n",
       "      <td>Bramor Enterprises Limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>432662</td>\n",
       "      <td>NAVEXIM S.A.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>524224</td>\n",
       "      <td>Magal Group SA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company_id                        name\n",
       "0  634022      PRIMCOM SA                \n",
       "1  324497      The David Isaacs Fund     \n",
       "2  280848      Bramor Enterprises Limited\n",
       "3  432662      NAVEXIM S.A.              \n",
       "4  524224      Magal Group SA            "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies = companies.reset_index(drop=True)\n",
    "companies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sample Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# orig_size = len(companies)\n",
    "# sample_size = 200000 \n",
    "# companies = companies.sample(sample_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_0 = train.loc[train[\"company_id\"]==-1]\n",
    "# print train_0.shape\n",
    "\n",
    "# train_1 = train.loc[train[\"company_id\"]>=0]\n",
    "# print train_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = train_1.loc[train_1[\"company_id\"].isin(companies[\"company_id\"])]\n",
    "# print train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train0_sample = train_0.sample(int(sample_size/float(orig_size)*len(train_0)))\n",
    "# print len(train0_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = pd.concat([train,train0_sample])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 3)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>TRATTAMENTO Ltd RIFIUTI METROPOLITANI SPA  SIGLABILE TRM SPA</td>\n",
       "      <td>177358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A IRL Fuund</td>\n",
       "      <td>568472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>BMR-500 Kendall LLC 1 Mezz GmbH</td>\n",
       "      <td>195692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Solich GmbH KG</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Drzyzzga Funds Logi. sp. z oo</td>\n",
       "      <td>404178</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_index                                                          name  \\\n",
       "0  0            TRATTAMENTO Ltd RIFIUTI METROPOLITANI SPA  SIGLABILE TRM SPA   \n",
       "1  1            A IRL Fuund                                                    \n",
       "2  2            BMR-500 Kendall LLC 1 Mezz GmbH                                \n",
       "3  3            Solich GmbH KG                                                 \n",
       "4  4            Drzyzzga Funds Logi. sp. z oo                                  \n",
       "\n",
       "   company_id  \n",
       "0  177358      \n",
       "1  568472      \n",
       "2  195692      \n",
       "3 -1           \n",
       "4  404178      "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.reset_index(drop=True)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make map of company_ids to Annoy_index_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# annoy_ids are from 0 to len(company_ids) -1 \n",
    "# annoy_ids are stored in list. List index are annoy_ids and the value is teh correspodnign company_id\n",
    "# Company_ids are stored in a dict. The key is company id and the value is annoy_id\n",
    "def build_company_annoy_maps(company_ids):\n",
    "    annoy2company = []\n",
    "    company2annoy = defaultdict(lambda:0)\n",
    "    for c_id in company_ids:\n",
    "        if c_id not in company2annoy:\n",
    "            annoy2company.append(c_id)\n",
    "            company2annoy[c_id] = len(annoy2company)-1\n",
    "    return annoy2company, company2annoy  \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "annoy2company, company2annoy = build_company_annoy_maps(companies[\"company_id\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "450256\n",
      "450256\n"
     ]
    }
   ],
   "source": [
    "print len(annoy2company)\n",
    "print len(company2annoy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34100 198710 34100\n",
      "4580 484544 4580\n",
      "30891 320028 30891\n",
      "42096 89805 42096\n",
      "41728 232717 41728\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "for i in range(5):\n",
    "    annoy_id = random.randint(0,50000)\n",
    "    assert annoy_id ==  company2annoy[annoy2company[annoy_id]]\n",
    "    print annoy_id, annoy2company[annoy_id], company2annoy[annoy2company[annoy_id]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_accents(df,**kw_args):\n",
    "    old_col =  kw_args[\"old_col\"]\n",
    "    new_col =  kw_args[\"new_col\"]\n",
    "    def remove_accents_inner(input_str):\n",
    "        nfkd_form = unicodedata.normalize('NFKD', unicode(input_str, 'utf8'))\n",
    "        return u\"\".join([c for c in nfkd_form if not unicodedata.combining(c)])\n",
    "    \n",
    "    df[new_col] = df[old_col].apply(remove_accents_inner)\n",
    "    return df\n",
    "\n",
    "def make_accent_transformer(old_col, new_col):\n",
    "    return FunctionTransformer(remove_accents, validate=False,\n",
    "                                         kw_args={\"old_col\":old_col,\"new_col\":new_col})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/akash/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "REPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]|@,;\\-\\\\\\+#~!$%^]')\n",
    "BAD_SYMBOLS_RE = re.compile('[^0-9a-z ]')\n",
    "STOPWORDS = set(stopwords.words('english'))\n",
    "#-#+_!`~\\\\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "india add g ddcbg c g hh ak ash sin gh dee pt su\n"
     ]
    }
   ],
   "source": [
    "s = \"for from india and add+g-ddc.b.g,c\\g-hh ak|ash#sin/gh dee$pt^a su%d\"\n",
    "s=re.sub(REPLACE_BY_SPACE_RE,\" \",s)\n",
    "s= re.sub(BAD_SYMBOLS_RE,\"\",s)\n",
    "s= ' '.join(word for word in s.split() if word not in STOPWORDS)\n",
    "print s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_name(df,**kw_args):\n",
    "    old_col =  kw_args[\"old_col\"]\n",
    "    new_col =  kw_args[\"new_col\"]\n",
    "    \n",
    "    def regex_clean(text):\n",
    "        text = re.sub(REPLACE_BY_SPACE_RE, \" \", text)\n",
    "        text = re.sub(BAD_SYMBOLS_RE, \"\", text)\n",
    "#         #add shorthand\n",
    "#         longhand = ' '.join(word for word in text.split() if not any(c.isdigit() for c in word))\n",
    "#         shorthand = \"\".join(word[0] for word in re.findall(\"\\w+\", longhand))\n",
    "#         text = text+\" \"+shorthand\n",
    "        return text\n",
    "\n",
    "    def strip_extra_spaces(text):\n",
    "        return \" \".join(text.split())\n",
    "    \n",
    "    df[new_col] = df[old_col].str.lower().str.strip()\n",
    "    df[new_col] = df[new_col].apply(regex_clean)\n",
    "    df[new_col] = df[new_col].apply(strip_extra_spaces)\n",
    "    return df\n",
    "\n",
    "def make_clean_name_transformer(old_col, new_col):\n",
    "    return FunctionTransformer(clean_name, validate=False, kw_args={\"old_col\":old_col,\"new_col\":new_col})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_pipeline = make_pipeline(make_accent_transformer(\"name\", \"clean_name\"),\n",
    "                                    make_clean_name_transformer(\"clean_name\", \"clean_name\"))\n",
    "companies = name_pipeline.transform(companies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_id</th>\n",
       "      <th>name</th>\n",
       "      <th>clean_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>116022</th>\n",
       "      <td>531186</td>\n",
       "      <td>EVI East Grand Forks LLC</td>\n",
       "      <td>evi east grand forks llc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281337</th>\n",
       "      <td>318011</td>\n",
       "      <td>Linden Logistics (Lux) S.à r.l.</td>\n",
       "      <td>linden logistics lux sa rl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348395</th>\n",
       "      <td>612231</td>\n",
       "      <td>Boryszewo Wind Invest spółka z ograniczoną odpowiedzialnością</td>\n",
       "      <td>boryszewo wind invest spoka z ograniczona odpowiedzialnoscia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>368424</th>\n",
       "      <td>82576</td>\n",
       "      <td>BASF Coatings Services S.A.</td>\n",
       "      <td>basf coatings services sa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242055</th>\n",
       "      <td>28080</td>\n",
       "      <td>PREFA PRAHA a.s.</td>\n",
       "      <td>prefa praha as</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51627</th>\n",
       "      <td>570672</td>\n",
       "      <td>Gesellschaft für staatsbürgerliche Bildung Saar mbH</td>\n",
       "      <td>gesellschaft fur staatsburgerliche bildung saar mbh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144787</th>\n",
       "      <td>108161</td>\n",
       "      <td>Fidelity Funds - Solutions SMART Defensive Pool</td>\n",
       "      <td>fidelity funds solutions smart defensive pool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145193</th>\n",
       "      <td>264180</td>\n",
       "      <td>HUNTSMAN P&amp;A SPAIN, S.L</td>\n",
       "      <td>huntsman pa spain sl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243976</th>\n",
       "      <td>408730</td>\n",
       "      <td>LOGICOR (CURVE) WE EINS GMBH</td>\n",
       "      <td>logicor curve we eins gmbh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299347</th>\n",
       "      <td>602125</td>\n",
       "      <td>Graben 19 Immobilien GmbH</td>\n",
       "      <td>graben 19 immobilien gmbh</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        company_id  \\\n",
       "116022  531186       \n",
       "281337  318011       \n",
       "348395  612231       \n",
       "368424  82576        \n",
       "242055  28080        \n",
       "51627   570672       \n",
       "144787  108161       \n",
       "145193  264180       \n",
       "243976  408730       \n",
       "299347  602125       \n",
       "\n",
       "                                                                 name  \\\n",
       "116022  EVI East Grand Forks LLC                                        \n",
       "281337  Linden Logistics (Lux) S.à r.l.                                 \n",
       "348395  Boryszewo Wind Invest spółka z ograniczoną odpowiedzialnością   \n",
       "368424  BASF Coatings Services S.A.                                     \n",
       "242055  PREFA PRAHA a.s.                                                \n",
       "51627   Gesellschaft für staatsbürgerliche Bildung Saar mbH             \n",
       "144787  Fidelity Funds - Solutions SMART Defensive Pool                 \n",
       "145193  HUNTSMAN P&A SPAIN, S.L                                         \n",
       "243976  LOGICOR (CURVE) WE EINS GMBH                                    \n",
       "299347  Graben 19 Immobilien GmbH                                       \n",
       "\n",
       "                                                          clean_name  \n",
       "116022  evi east grand forks llc                                      \n",
       "281337  linden logistics lux sa rl                                    \n",
       "348395  boryszewo wind invest spoka z ograniczona odpowiedzialnoscia  \n",
       "368424  basf coatings services sa                                     \n",
       "242055  prefa praha as                                                \n",
       "51627   gesellschaft fur staatsburgerliche bildung saar mbh           \n",
       "144787  fidelity funds solutions smart defensive pool                 \n",
       "145193  huntsman pa spain sl                                          \n",
       "243976  logicor curve we eins gmbh                                    \n",
       "299347  graben 19 immobilien gmbh                                     "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(450256, 3)\n",
      "448050\n"
     ]
    }
   ],
   "source": [
    "print companies.shape\n",
    "print companies[\"clean_name\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>TRATTAMENTO Ltd RIFIUTI METROPOLITANI SPA  SIGLABILE TRM SPA</td>\n",
       "      <td>177358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A IRL Fuund</td>\n",
       "      <td>568472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>BMR-500 Kendall LLC 1 Mezz GmbH</td>\n",
       "      <td>195692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Solich GmbH KG</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Drzyzzga Funds Logi. sp. z oo</td>\n",
       "      <td>404178</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_index                                                          name  \\\n",
       "0  0            TRATTAMENTO Ltd RIFIUTI METROPOLITANI SPA  SIGLABILE TRM SPA   \n",
       "1  1            A IRL Fuund                                                    \n",
       "2  2            BMR-500 Kendall LLC 1 Mezz GmbH                                \n",
       "3  3            Solich GmbH KG                                                 \n",
       "4  4            Drzyzzga Funds Logi. sp. z oo                                  \n",
       "\n",
       "   company_id  \n",
       "0  177358      \n",
       "1  568472      \n",
       "2  195692      \n",
       "3 -1           \n",
       "4  404178      "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>87480</th>\n",
       "      <td>87480</td>\n",
       "      <td>Aberd een GA Property Share Fdund</td>\n",
       "      <td>514345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83155</th>\n",
       "      <td>83155</td>\n",
       "      <td>Inter. Capital SEC</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89655</th>\n",
       "      <td>89655</td>\n",
       "      <td>VM, LC Holdings</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61920</th>\n",
       "      <td>61920</td>\n",
       "      <td>HIDROTECNICA IMMOBILIARE SRL</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91770</th>\n",
       "      <td>91770</td>\n",
       "      <td>ASSETalliane LIMITED LEASING</td>\n",
       "      <td>368975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12387</th>\n",
       "      <td>12387</td>\n",
       "      <td>RESOURCES DAYTALspain, SL</td>\n",
       "      <td>161446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86495</th>\n",
       "      <td>86495</td>\n",
       "      <td>Landeskirche/LB AM 95 ZVK NROD Hanno.</td>\n",
       "      <td>380644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27839</th>\n",
       "      <td>27839</td>\n",
       "      <td>Jona</td>\n",
       "      <td>627708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93834</th>\n",
       "      <td>93834</td>\n",
       "      <td>300 Rke Drive, LLC</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76313</th>\n",
       "      <td>76313</td>\n",
       "      <td>VantageTrust Master Coll ectiveiii Funds Trust - VT Vantagepoint Emerging Markets Fund</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       train_index  \\\n",
       "87480  87480         \n",
       "83155  83155         \n",
       "89655  89655         \n",
       "61920  61920         \n",
       "91770  91770         \n",
       "12387  12387         \n",
       "86495  86495         \n",
       "27839  27839         \n",
       "93834  93834         \n",
       "76313  76313         \n",
       "\n",
       "                                                                                         name  \\\n",
       "87480  Aberd een GA Property Share Fdund                                                        \n",
       "83155  Inter. Capital SEC                                                                       \n",
       "89655  VM, LC Holdings                                                                          \n",
       "61920  HIDROTECNICA IMMOBILIARE SRL                                                             \n",
       "91770  ASSETalliane LIMITED LEASING                                                             \n",
       "12387  RESOURCES DAYTALspain, SL                                                                \n",
       "86495  Landeskirche/LB AM 95 ZVK NROD Hanno.                                                    \n",
       "27839  Jona                                                                                     \n",
       "93834  300 Rke Drive, LLC                                                                       \n",
       "76313  VantageTrust Master Coll ectiveiii Funds Trust - VT Vantagepoint Emerging Markets Fund   \n",
       "\n",
       "       company_id  \n",
       "87480  514345      \n",
       "83155 -1           \n",
       "89655 -1           \n",
       "61920 -1           \n",
       "91770  368975      \n",
       "12387  161446      \n",
       "86495  380644      \n",
       "27839  627708      \n",
       "93834 -1           \n",
       "76313 -1           "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print train.shape\n",
    "train.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "count_vectorizer = CountVectorizer(analyzer='word', ngram_range=(1, 1), min_df=1)\n",
    "# X matrix where the row represents sentences and column is our one-hot vector for each token in our vocabulary\n",
    "X = count_vectorizer.fit_transform(companies[\"clean_name\"])\n",
    "\n",
    "# Vocabulary\n",
    "vocab = list(count_vectorizer.get_feature_names())\n",
    "\n",
    "# Column-wise sum of the X matrix.\n",
    "# It's some crazy numpy syntax that looks horribly unpythonic\n",
    "# For details, see http://stackoverflow.com/questions/3337301/numpy-matrix-to-array\n",
    "# and http://stackoverflow.com/questions/13567345/how-to-calculate-the-sum-of-all-columns-of-a-2d-numpy-array-efficiently\n",
    "counts = X.sum(axis=0).A1\n",
    "\n",
    "freq_distribution = Counter(dict(zip(vocab, counts)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'llc', 44288), (u'fund', 38551), (u'limited', 34803), (u'gmbh', 29272), (u'inc', 24952), (u'bv', 21019), (u'srl', 20646), (u'trust', 20440), (u'ltd', 16548), (u'sa', 15202), (u'co', 12984), (u'lp', 11681), (u'global', 11272), (u'de', 10292), (u'the', 10220), (u'company', 10159), (u'funds', 9906), (u'kg', 9865), (u'holding', 9770), (u'ab', 9459), (u'investment', 8335), (u'sl', 7666), (u'capital', 7485), (u'international', 7283), (u'equity', 6886), (u'di', 6700), (u'holdings', 6683), (u'of', 6680), (u'spa', 6663), (u'group', 6212), (u'as', 6092), (u'ii', 6061), (u'sicav', 5883), (u'management', 5723), (u'investments', 5456), (u'aps', 5192), (u'portfolio', 4959), (u'bond', 4854), (u'partners', 4848), (u'societa', 4605), (u'invest', 4526), (u'and', 4211), (u'oy', 4152), (u'sro', 4143), (u'bank', 3981), (u'plc', 3861), (u'services', 3851), (u'income', 3848), (u'master', 3799), (u'corporation', 3747), (u'nv', 3573), (u'beheer', 3568), (u'mbh', 3509), (u'spoka', 3506), (u'ag', 3479), (u'settlement', 3331), (u'series', 3274), (u'asset', 3227), (u'etf', 3223), (u'fonds', 3183), (u'europe', 3182), (u'credit', 3158), (u'us', 3081), (u'und', 3062), (u'uk', 3029), (u'properties', 3014), (u'sp', 2970), (u'sas', 2925), (u'in', 2895), (u'index', 2853), (u'energy', 2847), (u'street', 2815), (u'finance', 2742), (u'snc', 2719), (u'real', 2631), (u'public', 2604), (u'markets', 2541), (u'rl', 2510), (u'aktiebolag', 2493), (u'pty', 2451), (u'emerging', 2417), (u'van', 2403), (u'multi', 2368), (u'trading', 2365), (u'iii', 2326), (u'growth', 2325), (u'estate', 2320), (u'tarsasag', 2298), (u'solutions', 2260), (u'korlatolt', 2237), (u'partnership', 2233), (u'felelossegu', 2225), (u'pension', 2175), (u'euro', 2142), (u'opportunities', 2138), (u'oo', 2129), (u'ograniczona', 2127), (u'odpowiedzialnoscia', 2116), (u'for', 2112), (u'property', 2076)]\n"
     ]
    }
   ],
   "source": [
    "print (freq_distribution.most_common(100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = name_pipeline.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "      <th>clean_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>59665</th>\n",
       "      <td>59665</td>\n",
       "      <td>ASH FARMS LD.</td>\n",
       "      <td>432998</td>\n",
       "      <td>ash farms ld</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9267</th>\n",
       "      <td>9267</td>\n",
       "      <td>SCAT</td>\n",
       "      <td>519237</td>\n",
       "      <td>scat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68974</th>\n",
       "      <td>68974</td>\n",
       "      <td>Summit Real-Estate Omega GmbH</td>\n",
       "      <td>313570</td>\n",
       "      <td>summit real estate omega gmbh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66352</th>\n",
       "      <td>66352</td>\n",
       "      <td>CEGELEC EE</td>\n",
       "      <td>390949</td>\n",
       "      <td>cegelec ee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52567</th>\n",
       "      <td>52567</td>\n",
       "      <td>Indium V (Mauritius) Hold ings Limited TRUST</td>\n",
       "      <td>419594</td>\n",
       "      <td>indium v mauritius hold ings limited trust</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96532</th>\n",
       "      <td>96532</td>\n",
       "      <td>2250 Fourth Avenue Fund Partnership</td>\n",
       "      <td>147223</td>\n",
       "      <td>2250 fourth avenue fund partnership</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76669</th>\n",
       "      <td>76669</td>\n",
       "      <td>Belfius Select Portf olio Capital SA - World Balanced 40</td>\n",
       "      <td>80502</td>\n",
       "      <td>belfius select portf olio capital sa world balanced 40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17103</th>\n",
       "      <td>17103</td>\n",
       "      <td>Wildwood FUND IP, LLC</td>\n",
       "      <td>-1</td>\n",
       "      <td>wildwood fund ip llc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24518</th>\n",
       "      <td>24518</td>\n",
       "      <td>TFC</td>\n",
       "      <td>426935</td>\n",
       "      <td>tfc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59208</th>\n",
       "      <td>59208</td>\n",
       "      <td>Wolk Holdings Aktiengesellschaft</td>\n",
       "      <td>1666</td>\n",
       "      <td>wolk holdings aktiengesellschaft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       train_index                                                      name  \\\n",
       "59665  59665        ASH FARMS LD.                                              \n",
       "9267   9267         SCAT                                                       \n",
       "68974  68974        Summit Real-Estate Omega GmbH                              \n",
       "66352  66352        CEGELEC EE                                                 \n",
       "52567  52567        Indium V (Mauritius) Hold ings Limited TRUST               \n",
       "96532  96532        2250 Fourth Avenue Fund Partnership                        \n",
       "76669  76669        Belfius Select Portf olio Capital SA - World Balanced 40   \n",
       "17103  17103        Wildwood FUND IP, LLC                                      \n",
       "24518  24518        TFC                                                        \n",
       "59208  59208        Wolk Holdings Aktiengesellschaft                           \n",
       "\n",
       "       company_id                                              clean_name  \n",
       "59665  432998      ash farms ld                                            \n",
       "9267   519237      scat                                                    \n",
       "68974  313570      summit real estate omega gmbh                           \n",
       "66352  390949      cegelec ee                                              \n",
       "52567  419594      indium v mauritius hold ings limited trust              \n",
       "96532  147223      2250 fourth avenue fund partnership                     \n",
       "76669  80502       belfius select portf olio capital sa world balanced 40  \n",
       "17103 -1           wildwood fund ip llc                                    \n",
       "24518  426935      tfc                                                     \n",
       "59208  1666        wolk holdings aktiengesellschaft                        "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "assert gensim.models.doc2vec.FAST_VERSION > -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_id</th>\n",
       "      <th>name</th>\n",
       "      <th>clean_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>634022</td>\n",
       "      <td>PRIMCOM SA</td>\n",
       "      <td>primcom sa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>324497</td>\n",
       "      <td>The David Isaacs Fund</td>\n",
       "      <td>the david isaacs fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>280848</td>\n",
       "      <td>Bramor Enterprises Limited</td>\n",
       "      <td>bramor enterprises limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>432662</td>\n",
       "      <td>NAVEXIM S.A.</td>\n",
       "      <td>navexim sa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>524224</td>\n",
       "      <td>Magal Group SA</td>\n",
       "      <td>magal group sa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>513585</td>\n",
       "      <td>Marly SPF S.A.</td>\n",
       "      <td>marly spf sa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>354496</td>\n",
       "      <td>I.T APPARELS LIMITED</td>\n",
       "      <td>it apparels limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>381944</td>\n",
       "      <td>VX 30.141 ApS</td>\n",
       "      <td>vx 30141 aps</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>526057</td>\n",
       "      <td>Rydex ETF Trust - Guggenheim S&amp;P 500 Equal Weight ETF</td>\n",
       "      <td>rydex etf trust guggenheim sp 500 equal weight etf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>34381</td>\n",
       "      <td>Rydex Series Funds - Retailing Fund</td>\n",
       "      <td>rydex series funds retailing fund</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company_id                                                   name  \\\n",
       "0  634022      PRIMCOM SA                                              \n",
       "1  324497      The David Isaacs Fund                                   \n",
       "2  280848      Bramor Enterprises Limited                              \n",
       "3  432662      NAVEXIM S.A.                                            \n",
       "4  524224      Magal Group SA                                          \n",
       "5  513585      Marly SPF S.A.                                          \n",
       "6  354496      I.T APPARELS LIMITED                                    \n",
       "7  381944      VX 30.141 ApS                                           \n",
       "8  526057      Rydex ETF Trust - Guggenheim S&P 500 Equal Weight ETF   \n",
       "9  34381       Rydex Series Funds - Retailing Fund                     \n",
       "\n",
       "                                           clean_name  \n",
       "0  primcom sa                                          \n",
       "1  the david isaacs fund                               \n",
       "2  bramor enterprises limited                          \n",
       "3  navexim sa                                          \n",
       "4  magal group sa                                      \n",
       "5  marly spf sa                                        \n",
       "6  it apparels limited                                 \n",
       "7  vx 30141 aps                                        \n",
       "8  rydex etf trust guggenheim sp 500 equal weight etf  \n",
       "9  rydex series funds retailing fund                   "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8    rydex etf trust guggenheim sp 500 equal weight etf\n",
       "Name: clean_name, dtype: object"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies.loc[companies[\"company_id\"] == 526057,\"clean_name\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([u'primcom sa', u'the david isaacs fund',\n",
       "       u'bramor enterprises limited', u'navexim sa', u'magal group sa',\n",
       "       u'marly spf sa', u'it apparels limited', u'vx 30141 aps',\n",
       "       u'rydex etf trust guggenheim sp 500 equal weight etf'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies.clean_name.values[0:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['stud', 'tude', 'uden', 'dent', 'ent ', 'nt d', 't dd', ' ddg', 'ddg ', 'dg d', 'g df', ' dfs']\n"
     ]
    }
   ],
   "source": [
    "b='student ddg dfs'\n",
    "n=4\n",
    "print [b[i:i+n] for i in range(len(b)-n+1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SETTINGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_grams=[2,3]\n",
    "dim = 400\n",
    "window =5\n",
    "min_count=5\n",
    "workers=4\n",
    "num_trees_annoy=300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_corpus(names, n_grams= [3]):\n",
    "    corpus = []\n",
    "    for name in names:\n",
    "        name_grams = []\n",
    "        for n in n_grams:\n",
    "            name_grams.extend([name[i:i+n] for i in range(len(name)-n+1)])\n",
    "        corpus.append(name_grams)\n",
    "    return corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_name_vectors(corpus):\n",
    "    name_vectors = np.empty(shape=(len(corpus),dim))\n",
    "    for idx, name in enumerate(corpus):\n",
    "        name_vector = name_to_vec(name, model.wv)\n",
    "        name_vectors[idx,:] = name_vector\n",
    "    return name_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def name_to_vec(name, embeddings, dim=dim):\n",
    "    \"\"\"\n",
    "        name: a string\n",
    "        embeddings: dict where the key is a word and a value is its' embedding\n",
    "        dim: size of the representation\n",
    "\n",
    "        result: vector representation for the question\n",
    "    \"\"\"\n",
    "    \n",
    "    result = np.zeros(shape=dim)\n",
    "    count = 0\n",
    "    for word in name:\n",
    "        if word in embeddings:\n",
    "            result = result + embeddings[word]\n",
    "            count+=1\n",
    "    \n",
    "    if count>0:\n",
    "        result = result/count\n",
    "        \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "company_corpus = make_corpus(companies[\"clean_name\"], n_grams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(company_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "company_corpus[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print gensim.models.doc2vec.FAST_VERSION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Word2Vec(company_corpus, size=dim, window=window, min_count=min_count, workers=workers)\n",
    "model.wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv[\"llc\"].size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "company_vectors = make_name_vectors(company_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "company_vectors[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# index_size = dim\n",
    "# table = AnnoyIndex(index_size)\n",
    "\n",
    "# for i in range(company_vectors.shape[0]):\n",
    "#     if i%20000==0:\n",
    "#         print \"indexed %s items\"%i\n",
    "#     table.add_item(i,company_vectors[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# table.build(num_trees_annoy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# table.save(\"annoy_w2vec_index.ann\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = AnnoyIndex(index_size)\n",
    "table.load(\"annoy_w2vec_index.ann\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reteiveal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.merge(companies, how=\"left\", on=\"company_id\", suffixes=('','_truth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_corpus = make_corpus(train[\"clean_name\"], n_grams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_corpus[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_vectors = tfidf_vectorizer.transform(train[\"clean_name\"])\n",
    "train_vectors = make_name_vectors(train_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_vectors[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_nearest_neighbours(vectors, num_neighbours=1, search_nodes=-1, include_distances=True):\n",
    "    neighbours = np.empty(shape=(len(vectors),num_neighbours), dtype=np.int32)\n",
    "    distances = np.empty(shape=(len(vectors),num_neighbours))\n",
    "#     print len(vectors)\n",
    "#     print neighbours.shape\n",
    "#     print distances.shape\n",
    "    for idx,v in enumerate(vectors):\n",
    "        annoy_ids, annoy_distances = table.get_nns_by_vector(v, n=num_neighbours, search_k=search_nodes, include_distances=include_distances)\n",
    "        neighbours[idx,:] = annoy_ids\n",
    "        distances[idx,:] = annoy_distances\n",
    "    return neighbours, distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neighbours, distances = get_nearest_neighbours(train_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neigbours =np.vectorize(lambda x: annoy2company[x])(neighbours)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rank_1 = neigbours[:,0]\n",
    "rank_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"rank1_id\"] = rank_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train= train.merge(companies[['company_id', 'clean_name']], left_on=\"rank1_id\", \n",
    "                   right_on=\"company_id\", suffixes=(\"\",\"_rank1\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print len(train)\n",
    "print train.loc[train[\"company_id\"]==train[\"rank1_id\"]].shape\n",
    "print train.loc[train[\"company_id\"]!=train[\"rank1_id\"]].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sub = train.loc[train[\"company_id\"]!=-1]\n",
    "print np.sum(train_sub[\"company_id\"] == train_sub[\"rank1_id\"])/float(len(train_sub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Accuracy = 0.5219\n",
    "* n_grams=[2,3] \n",
    "* dim =300\n",
    "* window =5\n",
    "* min_count=5\n",
    "* workers=4\n",
    "* num_trees_annoy=300\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Process Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = name_pipeline.transform(test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}