segmenters test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparison of chinese segmenters\n",
    "\n",
    "I compare precision of two chinese segmenters:\n",
    " - [stanford nlp chinese segmenter](https://nlp.stanford.edu/software/segmenter.html)\n",
    " - [jieba](https://github.com/fxsjy/jieba) python package (java and rust implementation are floating around on github)\n",
    " \n",
    "Segmentation was compared against this dataset:\n",
    "https://www.microsoft.com/en-us/download/details.aspx?id=52531\n",
    "\n",
    "Segmentation_distance between two segmentations is calculated int wo steps:\n",
    " - Convert segmented sentences into list of indices of word boundaries (\"I am Groot\" would be converted into `[1,3]`),\n",
    " - Calculate Levenstein distance between two lists of word boundaries. Levenstein distance is minimal number of insertions, deletions or substitutions needed to convert one list to another."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "import nltk\n",
    "from collections import defaultdict\n",
    "from utils import segmentation_distance\n",
    "from Dataset import parse_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences=parse_dataset('msra-chinese-word-segmentation-data-v1/msra_bakeoff3_training.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46364"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install stanfordCoreNlp as described here https://github.com/ellie-icekler/StanfordCoreNLP-Chinese/blob/master/README.md\n",
    "\n",
    "Start service by:\n",
    "\n",
    "java -Xmx4g -cp \"*\" edu.stanford.nlp.pipeline.StanfordCoreNLPServer -serverProperties StanfordCoreNLP-chinese.properties -preload tokenize,ssplit,pos,lemma,ner,parse -status_port 9001  -port 9001 -timeout 15000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['我家', '没有', '电脑', '。']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize.stanford import CoreNLPParser\n",
    "sttok = CoreNLPParser('http://localhost:9001')\n",
    "print(list(sttok.tokenize(u'我家没有电脑。')))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jieba_counts=defaultdict(int)\n",
    "stanford_counts=defaultdict(int)\n",
    "lengthsum=0\n",
    "for n,s in enumerate(sentences):\n",
    "    lengthsum+=len(s)-1\n",
    "    fullsentence=\"\".join(s)\n",
    "    jieba_cut=jieba.lcut(fullsentence)\n",
    "    stanford_cut=list(sttok.tokenize(fullsentence))\n",
    "    jieba_counts[segmentation_distance(s,jieba_cut)]+=1\n",
    "    stanford_counts[segmentation_distance(s,stanford_cut)]+=1\n",
    "    #if not n%100:\n",
    "    #    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average boundaries in a sentence:  26.309313260288153\n",
      "average jieba distance:  4.380726425675093 ,   ratio: 0.16650858128963236\n",
      "average stanford distance:  3.4891294970235527 ,   ratio: 0.1326195580441136\n",
      "distance: jieba    stanford\n",
      "0  :      4622      4843\n",
      "1  :      7516      8265\n",
      "2  :      7527      8466\n",
      "3  :      5991      7102\n",
      "4  :      4796      5329\n",
      "5  :      3683      3767\n",
      "6  :      2812      2593\n",
      "7  :      2062      1844\n",
      "8  :      1638      1273\n",
      "9  :      1283      827\n",
      "10  :      916      577\n",
      "11  :      690      423\n",
      "12  :      577      274\n",
      "13  :      440      199\n",
      "14  :      344      163\n",
      "15  :      250      100\n",
      "16  :      207      75\n",
      "17  :      180      56\n",
      "18  :      147      43\n",
      "19  :      97      25\n",
      "20  :      90      19\n",
      "21  :      83      14\n",
      "22  :      63      6\n",
      "23  :      46      6\n",
      "24  :      49      6\n",
      "25  :      39      9\n",
      "26  :      19      6\n",
      "27  :      25      6\n",
      "28  :      11      1\n",
      "29  :      22      0\n",
      "30  :      14      2\n",
      "31  :      12      0\n",
      "32  :      10      1\n",
      "33  :      11      1\n",
      "34  :      13      0\n",
      "35  :      3      2\n",
      "36  :      3      0\n",
      "37  :      7      0\n",
      "38  :      2      1\n",
      "39  :      0      1\n",
      "40  :      3      1\n",
      "41  :      4      0\n",
      "42  :      1      0\n",
      "43  :      3      0\n",
      "44  :      2      1\n",
      "45  :      2      0\n",
      "46  :      4      0\n",
      "47  :      1      0\n",
      "48  :      1      0\n",
      "49  :      0      0\n",
      "50  :      0      0\n",
      "51  :      0      0\n",
      "52  :      4      0\n",
      "53  :      0      1\n",
      "54  :      0      1\n",
      "55  :      0      0\n",
      "56  :      1      0\n",
      "57  :      0      0\n",
      "58  :      0      0\n",
      "59  :      1      4\n",
      "60  :      1      0\n",
      "61  :      0      0\n",
      "62  :      0      0\n",
      "63  :      0      0\n",
      "64  :      2      0\n",
      "65  :      0      0\n",
      "66  :      0      0\n",
      "67  :      1      0\n",
      "68  :      0      0\n",
      "69  :      0      0\n",
      "70  :      0      0\n",
      "71  :      0      0\n",
      "72  :      0      0\n",
      "73  :      0      0\n",
      "74  :      0      0\n",
      "75  :      0      0\n",
      "76  :      0      0\n",
      "77  :      0      0\n",
      "78  :      0      0\n",
      "79  :      0      0\n",
      "80  :      0      0\n",
      "81  :      0      0\n",
      "82  :      0      1\n",
      "83  :      0      0\n",
      "84  :      0      0\n",
      "85  :      0      0\n",
      "86  :      0      0\n",
      "87  :      1      0\n",
      "88  :      0      0\n",
      "89  :      1      0\n",
      "90  :      0      0\n",
      "91  :      0      0\n",
      "92  :      0      0\n",
      "93  :      0      0\n",
      "94  :      0      0\n",
      "95  :      0      1\n",
      "96  :      0      2\n",
      "97  :      0      0\n",
      "98  :      0      2\n",
      "99  :      0      2\n",
      "100  :      0      2\n"
     ]
    }
   ],
   "source": [
    "avrg_j=sum([k*v for k,v in jieba_counts.items()])/sum(jieba_counts.values())\n",
    "avrg_s=sum([k*v for k,v in stanford_counts.items()])/sum(stanford_counts.values())\n",
    "avrg_w=lengthsum/(n+1)\n",
    "print('average boundaries in a sentence: ',avrg_w)\n",
    "print('average jieba distance: ',avrg_j,',   ratio:',avrg_j/avrg_w)\n",
    "print('average stanford distance: ',avrg_s,',   ratio:',avrg_s/avrg_w)\n",
    "print(\"distance: jieba    stanford\")\n",
    "for k in range(101):\n",
    "    print(k,' :     ',jieba_counts[k],'    ',stanford_counts[k])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=['我们', '变', '而', '以', '书会友', '，', '以', '书', '结缘', '，', '把', '欧美']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "b=jieba.lcut(''.join(a))\n",
    "c=list(sttok.tokenize(''.join(a)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['我们', '变', '而', '以', '书会友', '，', '以', '书', '结缘', '，', '把', '欧美']\n",
      "['我们', '变而', '以书', '会友', '，', '以书', '结缘', '，', '把', '欧美']\n",
      "['我们', '变', '而', '以', '书', '会友', '，', '以', '书', '结缘', '，', '把', '欧美']\n"
     ]
    }
   ],
   "source": [
    "print(a)\n",
    "print(b)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "segmentation_distance(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "segmentation_distance(a,c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}