notebooks/class_14.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.display import HTML\n",
    "import jieba\n",
    "\n",
    "class SearcherIIndex():\n",
    "    \"\"\"倒排索引文本搜索实现类\n",
    "    \n",
    "    用倒排索引\n",
    "    利用Python的集合运算，来实现候选结果集之间交、并运算\n",
    "    \n",
    "    Attributes:\n",
    "        index: 检索使用的倒排索引\n",
    "        max_id: 当前索引的文档最大ID\n",
    "        doc_list: 索引文档原文8\n",
    "    \"\"\"\n",
    "    def __init__(self, docs_file): \n",
    "        \"\"\"初始化，用文件中的文本行构建倒排索引\n",
    "        \n",
    "        Args:\n",
    "            docs_file:包含带索引文档(文本)的文件名\n",
    "            \n",
    "        \"\"\"\n",
    "        self.index = dict()    \n",
    "        self.max_id = 0\n",
    "        self.doc_list = [] \n",
    "        \n",
    "        with open(docs_file, 'r') as f:\n",
    "            docs_data = f.read()\n",
    "        \n",
    "        for doc in docs_data.split():\n",
    "            self.add_doc(doc)\n",
    "\n",
    "    def add_doc(self, doc):\n",
    "        \"\"\"向索引中添加新文档\n",
    "        \n",
    "        Args:\n",
    "            doc:待检索的文档(文本)\n",
    "        \n",
    "        Returns:\n",
    "            新增文档ID\n",
    "        \"\"\"\n",
    "        self.doc_list.append(doc)\n",
    "        for term in list(jieba.cut_for_search(doc)):\n",
    "            #构建和更新各Term对应的Posting(集合)\n",
    "            if term in self.index: \n",
    "                self.index[term].add(self.max_id)\n",
    "            else:\n",
    "                self.index[term] = set([self.max_id])\n",
    "        self.max_id += 1\n",
    "        return self.max_id - 1\n",
    "    \n",
    "    def word_match(self, word):\n",
    "        \"\"\"从倒排索引中获取包含word的候选文档ID集合\n",
    "        \n",
    "        Args:\n",
    "            word:待检索的词(短语)\n",
    "            \n",
    "        Returns：\n",
    "            包含待检索词(短语)的文档ID集合\n",
    "        \"\"\"\n",
    "        result = None\n",
    "        for term in list(jieba.cut(word)):\n",
    "            if result is None:\n",
    "                result = self.index.get(term, set())\n",
    "            else:\n",
    "                result = result & self.index.get(term, set())\n",
    "        if result is None:\n",
    "            result = set()\n",
    "        return result\n",
    "\n",
    "    def conv_query(self, query):\n",
    "        \"\"\"将用户的查询转换成用eval可运行、返回结果ID集合的代码段\n",
    "        \n",
    "        Args:\n",
    "            query:待转换的原始查询字符串\n",
    "        \n",
    "        Returns:\n",
    "            转换完成可通过eval执行返回ID集合的代码段字符串\n",
    "        \"\"\"\n",
    "        query_new_parts = []\n",
    "        all_parts = list(jieba.cut(query))\n",
    "        idx = 0\n",
    "        cache = '' #缓存变量，用于回收分词过程被切开的短语片段\n",
    "        count_parts = len(all_parts)\n",
    "        while idx < count_parts:\n",
    "            if all_parts[idx] == '(' or all_parts[idx] == ')':\n",
    "                query_new_parts.append(all_parts[idx])\n",
    "            elif all_parts[idx] == ' ':\n",
    "                query_new_parts.append(' ')\n",
    "            elif all_parts[idx] in ('and', 'AND', '+'):\n",
    "                query_new_parts.append('&')\n",
    "            elif all_parts[idx] in ('or', 'OR'):\n",
    "                query_new_parts.append('|')\n",
    "            elif all_parts[idx] in ('not', 'NOT', '-'):\n",
    "                query_new_parts.append('-')\n",
    "            elif (idx + 1 < count_parts #被分词切开的短语部分回收至缓存\n",
    "                  and all_parts[idx+1] not in (' ', ')')): \n",
    "                cache += all_parts[idx]\n",
    "            elif (idx + 2 < count_parts #处理词间空格的形式\n",
    "                  and all_parts[idx+1] == \" \" \n",
    "                  and all_parts[idx+2] not in ('(', ')', 'and', 'AND', '+', 'or', 'OR', 'NOT', 'not', '+', '-', ' ')): \n",
    "                query_new_parts.append(\"self.word_match('{}') & \".format(all_parts[idx]))\n",
    "                idx += 2\n",
    "                continue\n",
    "            else:\n",
    "                query_new_parts.append(\"self.word_match('{}')\".format(cache + all_parts[idx]))\n",
    "                cache = '' #合并完成清空缓存\n",
    "            idx += 1\n",
    "        query_new = ''.join(query_new_parts)\n",
    "        return query_new\n",
    "\n",
    "    def highlighter(self, doc, word):\n",
    "        \"\"\"用word对doc进行HTML高亮\n",
    "        \n",
    "        Args:\n",
    "            doc:需要高亮的文档\n",
    "            word:要进行高亮的关键词(查询)\n",
    "            \n",
    "        Returns:\n",
    "            返回对关键词(查询)进行高亮的文档\n",
    "        \"\"\"\n",
    "        for part in list(jieba.cut(word)):\n",
    "            #TODO(CHG):短语高亮需要先分词\n",
    "            if part not in ('(', ')', 'and', 'AND', 'or', 'OR', 'NOT', 'not', ' '):\n",
    "                doc = doc.replace(part, '<span style=\"color:red\">{}</span>'.format(part))\n",
    "        return doc\n",
    "\n",
    "    def search(self, query):\n",
    "        \"\"\"用query进行查询返回结果文档列表\n",
    "        \n",
    "        Args:\n",
    "            query:用户的(复合)布尔查询字符串\n",
    "            \n",
    "        Returns:\n",
    "            复合查询要求的(高亮)文档结果列表\n",
    "        \"\"\"\n",
    "        result = []\n",
    "        query_new = self.conv_query(query)\n",
    "        for did in eval(query_new):\n",
    "            result.append(self.highlighter(self.doc_list[did], query))\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "\n",
    "class SearcherIIndexVII(SearcherIIndex):\n",
    "    \"\"\"倒排索引文本搜索实现类(改进)\n",
    "    \n",
    "    自定义解析，保留英文片段，将中文片段多粒度分词处理\n",
    "    \n",
    "    Attributes:\n",
    "        index: 检索使用的倒排索引\n",
    "        max_id: 当前索引的文档最大ID\n",
    "        doc_list: 索引文档原文\n",
    "    \"\"\"\n",
    "    def parse_doc(self, doc):\n",
    "        \"\"\"对文档进行自定义解析，保留英文串，对中文串多粒度分词\n",
    "        \n",
    "        Args:\n",
    "            doc:待解析的原始文档\n",
    "        \n",
    "        Returns:\n",
    "            解析结果列表，元素是切分得到的term\n",
    "        \"\"\"\n",
    "        result = []\n",
    "        state_last = ''\n",
    "        cache = ''\n",
    "        for c in doc:\n",
    "            state_c = c in string.ascii_letters \\\n",
    "                or c.isdigit() \\\n",
    "                or c in ('-', ':', '.')\n",
    "            if c == ' ':\n",
    "                if state_last:\n",
    "                    result.append(cache)\n",
    "                else:\n",
    "                    result.extend(list(jieba.cut_for_search(cache)))\n",
    "                result.append(' ')\n",
    "                cache = ''\n",
    "                state_last = '' \n",
    "            else:\n",
    "                if state_c == state_last:\n",
    "                    cache += c\n",
    "                else:\n",
    "                    if state_last != '':\n",
    "                        if state_last:\n",
    "                            result.append(cache)\n",
    "                        else:\n",
    "                            result.extend(list(jieba.cut_for_search(cache)))\n",
    "                    cache = c\n",
    "                state_last = state_c\n",
    "        if cache:\n",
    "            if state_last:\n",
    "                result.append(cache)\n",
    "            else:\n",
    "                result.extend(list(jieba.cut_for_search(cache)))\n",
    "        return result\n",
    "    \n",
    "    def add_doc(self, doc):\n",
    "        \"\"\"向索引中添加新文档(正常索引及ngram索引)\n",
    "        \n",
    "        Args:\n",
    "            doc:待检索的文档(文本)\n",
    "        \n",
    "        Returns:\n",
    "            新增文档ID\n",
    "        \"\"\"\n",
    "        self.doc_list.append(doc)\n",
    "        doc = doc.lower()\n",
    "        for term in self.parse_doc(doc):\n",
    "            #构建和更新各Term对应的Posting(集合)\n",
    "            if term in self.index: \n",
    "                self.index[term].add(self.max_id)\n",
    "            else:\n",
    "                self.index[term] = set([self.max_id])\n",
    "        \n",
    "        #构建ngram索引(以二元为例)\n",
    "        doclen = len(doc)\n",
    "        for i in range(doclen-1):\n",
    "            term = doc[i:i+2]\n",
    "            if term in self.index_b: \n",
    "                self.index_b[term].add(self.max_id)\n",
    "            else:\n",
    "                self.index_b[term] = set([self.max_id])\n",
    "                \n",
    "        self.max_id += 1\n",
    "        return self.max_id - 1\n",
    "    \n",
    "    def dumpIndex(self):\n",
    "        \"\"\"原样输出索引，用于检查索引构建结果\n",
    "        \n",
    "        Returns:\n",
    "            对索引(字典结构)的Dump输出\n",
    "        \"\"\"\n",
    "        print(self.index)\n",
    "    \n",
    "    def conv_query(self, query):\n",
    "        \"\"\"将用户的查询转换成用eval可运行、返回结果ID集合的代码段\n",
    "        \n",
    "        Args:\n",
    "            query:待转换的原始查询字符串\n",
    "        \n",
    "        Returns:\n",
    "            转换完成可通过eval执行返回ID集合的代码段字符串\n",
    "        \"\"\"\n",
    "        query_new_parts = []\n",
    "        all_parts = list(self.parse_query(query))\n",
    "        idx = 0\n",
    "        cache = '' #缓存变量，用于回收分词过程被切开的短语片段\n",
    "        count_parts = len(all_parts)\n",
    "        while idx < count_parts:\n",
    "            if all_parts[idx][1:] == '(' or all_parts[idx][1:] == ')':\n",
    "                query_new_parts.append(all_parts[idx][1:])\n",
    "            elif all_parts[idx][1:] == ' ':\n",
    "                query_new_parts.append(' ')\n",
    "            elif all_parts[idx][1:] in ('and', 'AND', '+'):\n",
    "                query_new_parts.append('&')\n",
    "            elif all_parts[idx][1:] in ('or', 'OR'):\n",
    "                query_new_parts.append('|')\n",
    "            elif all_parts[idx][1:] in ('not', 'NOT', '-'):\n",
    "                query_new_parts.append('-')\n",
    "            elif (idx + 1 < count_parts #对连续的内容分段结果集合中间加”&“运算符\n",
    "                  and all_parts[idx+1][1:] not in (' ', ')')): \n",
    "                query_new_parts.append(\"{} & \".format(self.conv_part(all_parts[idx])))\n",
    "            elif (idx + 2 < count_parts #处理词间、词与符号间空格的情况\n",
    "                  and all_parts[idx+1][1:] == \" \" \n",
    "                  and all_parts[idx+2][1:] not in (')', 'and', 'AND', '+', 'or', 'OR', 'NOT', 'not', '+', '-', ' ')): \n",
    "                query_new_parts.append(\"{} & \".format(self.conv_part(all_parts[idx])))\n",
    "                idx += 2\n",
    "                continue\n",
    "            else:\n",
    "                query_new_parts.append(self.conv_part(cache + all_parts[idx]))\n",
    "                cache = '' #合并完成清空缓存\n",
    "            idx += 1\n",
    "        query_new = ''.join(query_new_parts)\n",
    "        return query_new\n",
    "    \n",
    "    def term_match(self, term):\n",
    "        \"\"\"在索引里找到term对应的posting集合\n",
    "        \n",
    "        Args:\n",
    "            term:要检索的词项\n",
    "            \n",
    "        Results:\n",
    "            term对应的posting集合\n",
    "        \"\"\"\n",
    "        return self.index.get(term, set()) \n",
    "    \n",
    "    def conv_part(self, part):\n",
    "        \"\"\"将带有类别标记的解析结果段 转化为 eval能进行计算的代码段\n",
    "        \n",
    "        Args:\n",
    "            part:带有类别标记的解析结果段\n",
    "            \n",
    "        Results:\n",
    "            eval能进行计算的代码段字符串(调用 term_match() 进行计算)\n",
    "        \"\"\"\n",
    "        flag = part[0]\n",
    "        if flag == 'e':\n",
    "            return \"self.term_match('{}')\".format(part[1:])\n",
    "        elif flag == 'c':\n",
    "            return \"(self.term_match('{}'))\".format(\n",
    "                \"') & self.term_match('\".join(jieba.cut(part[1:])))\n",
    "        elif flag == 'f':\n",
    "            return \"self.frag_match('{}')\".format(part[1:])\n",
    "    \n",
    "    def highlighter(self, doc, query):\n",
    "        \"\"\"用query对doc进行HTML高亮\n",
    "        \n",
    "        Args:\n",
    "            doc:需要高亮的文档\n",
    "            word:要进行高亮的关键词(查询)\n",
    "            \n",
    "        Returns:\n",
    "            返回对关键词(查询)进行高亮的文档\n",
    "        \"\"\"\n",
    "        n = 0\n",
    "        #生成要进行高亮的关键词串集合\n",
    "        word_set = set()\n",
    "        query = query.lower()\n",
    "        query_parts = self.parse_query(query)\n",
    "        for query_part in query_parts:\n",
    "            if query_part[0] == 'e' or query_part[0] == 'f':\n",
    "                word_set.add(query_part[1:])\n",
    "                if len(query_part[1:]) > n:\n",
    "                    n = len(query_part[1:])\n",
    "            elif query_part[0] == 'c':\n",
    "                if len(query_part[1:]) > 1:\n",
    "                    for word in jieba.cut(query_part[1:]):\n",
    "                        word_set.add(word)\n",
    "                        if len(word) > n:\n",
    "                            n = len(word)\n",
    "        \n",
    "        #遍历文档替换高亮关键词串\n",
    "        doc_low = doc.lower()\n",
    "        i = 0\n",
    "        result = []\n",
    "        while True:\n",
    "            end_idx = i + n\n",
    "            if end_idx > len(doc_low):\n",
    "                end_idx = len(doc_low)\n",
    "            for j in range(end_idx, i, -1):\n",
    "                if doc_low[i:j] in word_set:\n",
    "                    break\n",
    "            if doc_low[i:j] in word_set:\n",
    "                result.append(\n",
    "                    '<span style=\"color:red\">{}</span>'.format(doc[i:j]))\n",
    "            else:\n",
    "                result.append(doc_low[i:j])\n",
    "            i = j\n",
    "            if i == len(doc_low):\n",
    "                break\n",
    "                \n",
    "        return ''.join(result)\n",
    "    \n",
    "    def __init__(self, docs_file): \n",
    "        \"\"\"初始化，用文件中的文本行构建倒排索引\n",
    "        \n",
    "        Args:\n",
    "            docs_file:包含带索引文档(文本)的文件名\n",
    "            \n",
    "        \"\"\"\n",
    "        self.index = dict() #标准倒排索引\n",
    "        self.index_b = dict() #ngram索引\n",
    "        self.max_id = 0\n",
    "        self.doc_list = [] \n",
    "        \n",
    "        with open(docs_file, 'r') as f:\n",
    "            docs_data = f.read()\n",
    "        \n",
    "        for doc in docs_data.split():\n",
    "            self.add_doc(doc)\n",
    "            \n",
    "    def frag_match(self, frag):\n",
    "        \"\"\"对片段frag用ngram索引实现原样搜索\n",
    "        \n",
    "        Args:\n",
    "            frag:要原样搜索的字符串\n",
    "            \n",
    "        Results:\n",
    "            片段原样搜索的结果(文档ID)集合\n",
    "        \"\"\"\n",
    "        frag = frag.lower() #大小写归一化\n",
    "        result = None\n",
    "        doclen = len(frag)\n",
    "        for i in range(doclen - 1):\n",
    "            term = frag[i:i+2]\n",
    "            if result is None:\n",
    "                result = self.index_b.get(term, set())\n",
    "            else:\n",
    "                result = result & self.index_b.get(term, set())\n",
    "        return result\n",
    "    \n",
    "    def get_char_type(self, c):\n",
    "        \"\"\"返回当前字符的类型(e,c,s,f,b)\n",
    "        \n",
    "        Args:\n",
    "            c:要进行判断的单个字符\n",
    "            \n",
    "        Results:\n",
    "            返回判断结果(前缀)：e为英文，c为中文，s为空格，f为引号，b为括号\n",
    "        \"\"\"\n",
    "        result = 'c'\n",
    "        if c in string.ascii_letters \\\n",
    "                or c.isdigit() \\\n",
    "                or c in ('-', ':', '.'):\n",
    "            result = 'e'\n",
    "        elif c == '\"':\n",
    "            result = 'f'\n",
    "        elif c == ' ':\n",
    "            result = 's'\n",
    "        elif c in ('(', ')'):\n",
    "            result = 'b'\n",
    "        return result\n",
    "    \n",
    "    def parse_query(self, doc):\n",
    "        \"\"\"对查询进行自定义解析，保留英文串，对中文串原型插入\n",
    "        \n",
    "        Args:\n",
    "            doc:待解析的原始文档\n",
    "        \n",
    "        Returns:\n",
    "            解析结果列表，元素是带有串类型标记(首字符，e为英文，c为中文，s为空格，f为引号，b为括号)\n",
    "            的切分term结果\n",
    "        \"\"\"\n",
    "        doc = doc.lower() + ' ' #解决末位字符状态切换问题的小技巧\n",
    "        result = []\n",
    "        doclen = len(doc)\n",
    "        i = 0\n",
    "        while True:\n",
    "            cur_char_type = self.get_char_type(doc[i])\n",
    "            for j in range(i+1, doclen):\n",
    "                if cur_char_type == 'f': #当前符号为引号，找下一个引号\n",
    "                    if self.get_char_type(doc[j]) == 'f':\n",
    "                        break\n",
    "                elif self.get_char_type(doc[j]) != cur_char_type: #当前符号非引号，找下一个状态变化\n",
    "                    break\n",
    "            if cur_char_type == 's': #对多个空格连续出现的情况进行合并\n",
    "                result.append('s ')\n",
    "            elif cur_char_type == 'f': #对引号只提取引号内字符串\n",
    "                result.append(cur_char_type + doc[i+1:j])\n",
    "                j += 1\n",
    "            else:\n",
    "                result.append(cur_char_type + doc[i:j])\n",
    "            i = j\n",
    "            if i >= doclen - 1:\n",
    "                break\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['f七连胜']\n",
      "['一边倒！中国女排3-0横扫美国取<span style=\"color:red\">七连胜</span>', '一边倒！中国女排3-0横扫美国取<span style=\"color:red\">七连胜</span>', '一边倒！中国女排3-0横扫美国取<span style=\"color:red\">七连胜</span>', '一边倒！中国女排3-0横扫美国取<span style=\"color:red\">七连胜</span>', '一边倒！中国女排3-0横扫美国取<span style=\"color:red\">七连胜</span>']\n"
     ]
    }
   ],
   "source": [
    "query = '\"七连胜\"'\n",
    "searcher = SearcherIIndexVII('titles.txt')\n",
    "print(searcher.parse_query(query))\n",
    "print(searcher.search(query))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fmate']\n",
      "self.frag_match('mate')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "外媒评<span style=\"color:red\">Mate</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style=\"color:red\">Mate</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "怼完苹果怼三星，刚发了<span style=\"color:red\">Mate</span>30的余承东依然要"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "华为<span style=\"color:red\">Mate</span>30采用安卓系统"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "可安装谷歌gms！华为<span style=\"color:red\">Mate</span>30确认支持boo"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style=\"color:red\">Mate</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "余承东:<span style=\"color:red\">Mate</span>30"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "iphone11和华为<span style=\"color:red\">Mate</span>30拍照对比：差距"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span style=\"color:red\">Mate</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "外媒评<span style=\"color:red\">Mate</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "怼完苹果怼三星，刚发了<span style=\"color:red\">Mate</span>30的余承东依然要"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "外媒评华为<span style=\"color:red\">Mate</span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "余承东:<span style=\"color:red\">Mate</span>30"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "华为<span style=\"color:red\">Mate</span>30采用安卓系统"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "华为发布<span style=\"color:red\">Mate</span>30系列手机：电池最大4500m"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "怼完苹果怼三星，刚发了<span style=\"color:red\">Mate</span>30的余承东依然要"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "searcher = SearcherIIndexVII('titles.txt')\n",
    "\n",
    "query = '\"mate\"'\n",
    "print(searcher.parse_query(query))\n",
    "print(searcher.conv_query(query))\n",
    "result = searcher.search(query)\n",
    "if result:\n",
    "    for doc in result:\n",
    "        display(HTML(doc))\n",
    "else:\n",
    "    print('No result.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 采集网易新闻排行新闻标题、URL列表 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "import csv\n",
    "import requests\n",
    "\n",
    "news_list = []\n",
    "url = 'http://news.163.com/rank/'\n",
    "r = requests.get(url) #下载\n",
    "sel = etree.HTML(r.text) #解析\n",
    "nodes = sel.xpath('//td/a')\n",
    "for node in nodes:\n",
    "    news_list.append([node.text, node.attrib['href']])\n",
    "    \n",
    "with open('163news.csv','w') as f: #入库\n",
    "    f_csv = csv.writer(f)\n",
    "    f_csv.writerows(news_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "南充小伙老家建洋气别墅 一栋自己住一栋送堂哥,https://home.163.com/19/1209/07/EVUGCN4I001081EI.html\n",
      "恋情实锤？杨幂魏大勋入住同一酒店没出来,https://ent.163.com/19/1210/08/F017MIA700038FO9.html\n",
      "奥巴马花8260万买豪宅 11.7万㎡大庄园拥私,https://home.163.com/19/1210/07/F012SVAN001081EI.html\n",
      "39岁董洁扮嫩演高中生 被吐槽比剧中姑妈还老气,https://ent.163.com/19/1209/07/EVUJ6ND000038FO9.html\n",
      "山西古村被“圈”收门票 九成都是危房只剩老人居住,https://home.163.com/19/1209/07/EVUGCSBR001081EI.html\n",
      "\"\"\"冰花男孩\"\"父亲申请贫困户遭拒 村主任:其名下有\",https://news.163.com/19/1209/09/EVUP0P590001899O.html\n",
      "五星级酒店竟是“毛坯房”?一晚9万6人们排队住,https://home.163.com/19/1209/07/EVUGCTIM001081EI.html\n",
      "心虚?李小璐晒旅游照又秒删 疑与PG One国外,https://ent.163.com/19/1209/09/EVUPOMRO00038FO9.html\n",
      "62岁老人自制创意鲁班家具 不用一颗钉子还可折叠,https://home.163.com/19/1210/07/F012SP99001081EI.html\n",
      "飞机起飞前乘客收到亲人噩耗 航班紧急滑回航站楼,https://news.163.com/19/1209/11/EVV0NUPF0001875P.html\n"
     ]
    }
   ],
   "source": [
    "!head 163news.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "#### 采集新闻正文"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://play.163.com/photoview/ITPG0031/95896.html\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "import csv\n",
    "import requests\n",
    "\n",
    "from tomorrow import threads\n",
    "\n",
    "#用tomorrow实现多线程采集\n",
    "@threads(5)\n",
    "def download(url):\n",
    "    return requests.get(url)\n",
    "\n",
    "news_list = []\n",
    "with open('163news.csv', 'r') as f:\n",
    "    f_csv = csv.reader(f)\n",
    "    for row in f_csv:\n",
    "        news_list.append([row[0], row[1]])\n",
    "        \n",
    "result_detail = []\n",
    "\n",
    "responses = [download(item[1]) for item in news_list[566:]]\n",
    "for r in responses:\n",
    "    url = r.url\n",
    "    sel = etree.HTML(r.text)\n",
    "    try:\n",
    "        title = sel.xpath(\"//title/text()\")[0]\n",
    "        if 'dy.163.com' in url:\n",
    "            nodes = sel.xpath(\"//div[@id='content']/p/text()\")\n",
    "        else:\n",
    "            nodes = sel.xpath(\"//div[@id='endText']/p/text()\")\n",
    "        content = ''.join(map(lambda x: x.strip(), nodes))\n",
    "        if(len(content) == 0):\n",
    "            print(url)\n",
    "            break\n",
    "    except Exception:\n",
    "        print(url)\n",
    "        break\n",
    "    result_detail.append([url, title, content])\n",
    "#     print(url, len(content))\n",
    "    \n",
    "with open('163news_d.csv','w') as f:\n",
    "    f_csv = csv.writer(f)\n",
    "    f_csv.writerows(result_detail)\n",
    "    \n",
    "print('Done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head 163news_d.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 继承新闻索引类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "class NewsIndex(SearcherIIndexVII):\n",
    "    \"\"\"新闻内容索引，实现对CSV文件的索引、对长文的高亮摘要、对新闻链接的支持\n",
    "    \"\"\"\n",
    "    def __init__(self, docs_file): \n",
    "        \"\"\"初始化，用CSV文件中的title+content构建倒排索引，将url保存在link_list里\n",
    "        \n",
    "        Args:\n",
    "            docs_file:包含新闻URL、title和content的csv文件的文件名\n",
    "            \n",
    "        \"\"\"\n",
    "        self.index = dict() #标准倒排索引\n",
    "        self.index_b = dict() #ngram索引\n",
    "        self.max_id = 0\n",
    "        self.doc_list = [] \n",
    "        self.link_list = []\n",
    "        \n",
    "        with open(docs_file, 'r') as f:\n",
    "            f_csv = csv.reader(f)\n",
    "            for row in f_csv:\n",
    "                self.add_doc(row[1] + row[2])\n",
    "                self.link_list.append(row[0])\n",
    "\n",
    "    def highlighter(self, doc, query):\n",
    "        \"\"\"用query对doc进行HTML高亮\n",
    "        \n",
    "        Args:\n",
    "            doc:需要高亮的文档\n",
    "            word:要进行高亮的关键词(查询)\n",
    "            \n",
    "        Returns:\n",
    "            返回对关键词(查询)进行高亮的文档\n",
    "        \"\"\"\n",
    "        n = 0\n",
    "        #生成要进行高亮的关键词串集合\n",
    "        word_set = set()\n",
    "        query = query.lower()\n",
    "        query_parts = self.parse_query(query)\n",
    "        for query_part in query_parts:\n",
    "            if query_part[0] == 'e' or query_part[0] == 'f':\n",
    "                word_set.add(query_part[1:])\n",
    "                if len(query_part[1:]) > n:\n",
    "                    n = len(query_part[1:])\n",
    "            elif query_part[0] == 'c':\n",
    "                if len(query_part[1:]) > 1:\n",
    "                    for word in jieba.cut(query_part[1:]):\n",
    "                        word_set.add(word)\n",
    "                        if len(word) > n:\n",
    "                            n = len(word)\n",
    "        \n",
    "        #遍历文档替换高亮关键词串\n",
    "        doc_low = doc.lower()\n",
    "        i = 0\n",
    "        result = []\n",
    "        side_len = 10 #设置上下文保留的宽度\n",
    "        last_end = 0 #上一个命中关键词的末位位置\n",
    "        while True:\n",
    "            end_idx = i + n\n",
    "            if end_idx > len(doc_low):\n",
    "                end_idx = len(doc_low)\n",
    "            for j in range(end_idx, i, -1):\n",
    "                if doc_low[i:j] in word_set:\n",
    "                    break\n",
    "            if doc_low[i:j] in word_set:\n",
    "                #追加从上一个关键词末位到当前关键词开头的周围上下文\n",
    "                if i - last_end > 2 * side_len:\n",
    "                    result.append(doc[last_end:last_end+side_len])\n",
    "                    result.append('...')\n",
    "                    result.append(doc[i-side_len:i])\n",
    "                else:\n",
    "                    result.append(doc[last_end:i])\n",
    "                #追加高亮处理的关键词串\n",
    "                result.append(\n",
    "                    '<span style=\"color:red\">{}</span>'.format(doc[i:j]))\n",
    "                last_end = j\n",
    "#             else:\n",
    "#                 result.append(doc_low[i:j])\n",
    "            i = j\n",
    "            if i == len(doc_low):\n",
    "                right_idx = last_end + side_len\n",
    "                if right_idx > len(doc):\n",
    "                    result.append(doc[last_end:])\n",
    "                else:\n",
    "                    result.append(doc[last_end:right_idx])\n",
    "                    result.append('...')\n",
    "                break\n",
    "                \n",
    "        return ''.join(result)\n",
    "    \n",
    "    def search(self, query):\n",
    "        \"\"\"用query进行查询返回结果文档列表\n",
    "        \n",
    "        Args:\n",
    "            query:用户的(复合)布尔查询字符串\n",
    "            \n",
    "        Returns:\n",
    "            复合查询要求的(高亮)文档结果列表\n",
    "        \"\"\"\n",
    "        result = []\n",
    "        query_new = self.conv_query(query)\n",
    "        for did in eval(query_new):\n",
    "            result.append([self.highlighter(self.doc_list[did], query), self.link_list[did]])\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_searcher = NewsIndex('163news_d.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href=\"https://sports.163.com/19/1203/04/EVEPDFHV00058781.html\" target=\"_blank\"><span style=\"color:red\">梅西</span>获颁2019年金球奖...上分量最重的金球奖由<span style=\"color:red\">梅西</span>获得。阿根廷人力压范...球》的名义单独评选。<span style=\"color:red\">梅西</span>和C罗是金球奖有史以...锋格里兹曼排在第三，<span style=\"color:red\">梅西</span>无缘三甲。2019年对于<span style=\"color:red\">梅西</span>而言是悲喜交加的一年...球先生。德罗巴宣布了<span style=\"color:red\">梅西</span>的获奖，上届金球得主莫德里奇为<span style=\"color:red\">梅西</span>颁出金球奖杯。<span style=\"color:red\">梅西</span>在2009年、201...金靴两项大奖。金球：<span style=\"color:red\">梅西</span>银球：范戴克铜球：C...</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "query = '梅西'\n",
    "result = news_searcher.search(query)\n",
    "if result:\n",
    "    for doc,url in result:\n",
    "        display(HTML('<a href=\"{}\" target=\"_blank\">{}</a>'.format(url,doc)))\n",
    "else:\n",
    "    print('No result.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tomorrow多线程采集示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 4.428255 seconds\n"
     ]
    }
   ],
   "source": [
    "urls = [\n",
    "    'http://google.com',\n",
    "    'http://facebook.com',\n",
    "    'http://youtube.com',\n",
    "    'http://baidu.com',\n",
    "    'http://yahoo.com',\n",
    "]\n",
    "\n",
    "import time\n",
    "import requests\n",
    "\n",
    "from tomorrow import threads\n",
    "\n",
    "@threads(5)\n",
    "def download(url):\n",
    "    return requests.get(url)\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\n",
    "    start = time.time()\n",
    "    responses = [download(url) for url in urls]\n",
    "    html = [response.text for response in responses]\n",
    "    end = time.time()\n",
    "    print(\"Time: %f seconds\" % (end - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### API采集示例 (参考 https://www.jianshu.com/p/c54e25349b77 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,30,10):\n",
    "    url = 'https://3g.163.com/touch/reconstruct/article/list/BA8D4A3Rwangning/{}-10.html'.format(i)\n",
    "    r = requests.get(url)\n",
    "    results = json.loads(r.text[9:-1])\n",
    "    for result in results['BA8D4A3Rwangning']:\n",
    "        print(result['title'], result['url'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 有些页面用默认设置无法采集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get('https://book.douban.com/tag/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "418"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.status_code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 采集有反爬虫机制的网页\n",
    "浏览器正常访问 -> Copy as Curl -> 粘贴至 https://curl.trillworks.com/ 转成Requests请求代码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cookies = {\n",
    "    \n",
    "}\n",
    "\n",
    "headers = {\n",
    "    'Connection': 'keep-alive',\n",
    "    'Cache-Control': 'max-age=0',\n",
    "    'Upgrade-Insecure-Requests': '1',\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',\n",
    "    'Sec-Fetch-User': '?1',\n",
    "    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',\n",
    "    'Sec-Fetch-Site': 'same-origin',\n",
    "    'Sec-Fetch-Mode': 'navigate',\n",
    "    'Referer': 'https://book.douban.com/',\n",
    "    'Accept-Encoding': 'gzip, deflate',\n",
    "    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',\n",
    "}\n",
    "\n",
    "response = requests.get('https://book.douban.com/tag/%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C', headers=headers, cookies=cookies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sel = etree.HTML(response.text) #解析\n",
    "nodes = sel.xpath(\"//li[@class='subject-item']\")\n",
    "for node in nodes:\n",
    "    title = node.xpath('.//h2/a/text()')[0].strip()\n",
    "    print(title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 罗辑思维文案的搜索与分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 搜索“罗辑思维 元无知”找到文案网址：https://www.ljsw.io/knowl/article/CG.html\n",
    "- 列表页如 https://www.ljsw.io/knowl/column/3 取正文链接的XPath为 '//div[@class='article-row']/div/a/@href'\n",
    "- 正文页如 https://www.ljsw.io/knowl/article/Da.html 取正文内容的XPath为 '//div[@class='custom-richtext']/div[@class='text']/p'\n",
    "- 将采集内容保存为 luojisiwei.csv 文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import math\n",
    "\n",
    "class LuoIndex(NewsIndex):\n",
    "    \"\"\"罗辑思维文案索引，实现对CSV文件的索引、对长文的高亮摘要、对文案标题、链接的支持\n",
    "    \"\"\"\n",
    "    def __init__(self, docs_file): \n",
    "        \"\"\"初始化，用CSV文件中的title、content构建倒排索引，将url保存在link_list里，title保存在title_list里\n",
    "        \n",
    "        Args:\n",
    "            docs_file:包含文案URL、title和content的csv文件的文件名\n",
    "            \n",
    "        \"\"\"\n",
    "        self.index = dict() #标准倒排索引\n",
    "        self.index_b = dict() #ngram索引\n",
    "        self.max_id = 0\n",
    "        self.doc_list = [] \n",
    "        self.link_list = []\n",
    "        self.title_list = []\n",
    "        \n",
    "        self.tf = [] #词频\n",
    "        self.df = defaultdict(int) #文档频率\n",
    "        \n",
    "        count = 0\n",
    "        with open(docs_file, 'r') as f:\n",
    "            f_csv = csv.reader(f)\n",
    "            for row in f_csv:\n",
    "                self.add_doc(row[1], row[2])\n",
    "                self.link_list.append(row[0])\n",
    "#                 count += 1\n",
    "#                 if count > 50:  #调试阶段限制文档数\n",
    "#                     break\n",
    "    \n",
    "    def add_doc(self, title, doc):\n",
    "        \"\"\"向索引中添加新文档(正常索引及ngram索引)\n",
    "        \n",
    "        Args:\n",
    "            title:待检索文档的标题(文本)\n",
    "            doc:待检索的文档(文本)\n",
    "        \n",
    "        Returns:\n",
    "            新增文档ID\n",
    "        \"\"\"\n",
    "        self.doc_list.append(doc)\n",
    "        self.title_list.append(title)\n",
    "        self.tf.append(defaultdict(int))\n",
    "        doc = (title + ' ' + doc).lower()\n",
    "        for term in self.parse_doc(doc):\n",
    "            #构建和更新各Term对应的Posting(集合)\n",
    "            if term in self.index: \n",
    "                self.index[term].add(self.max_id)\n",
    "            else:\n",
    "                self.index[term] = set([self.max_id])\n",
    "            self.tf[self.max_id][term] += 1\n",
    "            self.df[term] += 1\n",
    "        \n",
    "        #构建ngram索引(以二元为例)\n",
    "        doclen = len(doc)\n",
    "        for i in range(doclen-1):\n",
    "            term = doc[i:i+2]\n",
    "            if term in self.index_b: \n",
    "                self.index_b[term].add(self.max_id)\n",
    "            else:\n",
    "                self.index_b[term] = set([self.max_id])\n",
    "                \n",
    "        self.max_id += 1\n",
    "        return self.max_id - 1\n",
    "    \n",
    "    def cal_score(self, keywords, did):\n",
    "        \"\"\"根据关键词计算文档得分\n",
    "        \n",
    "        Args:\n",
    "            keywords:从查询中提取的打分关键词\n",
    "            did:待评分的文档(ID)\n",
    "        \n",
    "        Returns:\n",
    "            文档得分\n",
    "        \"\"\"\n",
    "        score = 0\n",
    "        for keyword in keywords:\n",
    "            score += self.tf[did].get(keyword, 0) / math.sqrt(len(self.doc_list[did])) / (self.df.get(keyword, 0) + 1)  #带长度惩罚的TFIDF得分\n",
    "        return score\n",
    "    \n",
    "    def search(self, query):\n",
    "        \"\"\"用query进行查询返回结果文档列表\n",
    "        \n",
    "        Args:\n",
    "            query:用户的(复合)布尔查询字符串\n",
    "            \n",
    "        Returns:\n",
    "            复合查询要求的(高亮)文档结果列表\n",
    "        \"\"\"\n",
    "        result = []\n",
    "        keywords, query_new = self.conv_query(query)\n",
    "        result_score = []\n",
    "        for did in eval(query_new):\n",
    "            result_score.append([did, self.cal_score(keywords, did)])\n",
    "        for did, score in sorted(result_score, key=lambda x: x[1], reverse=True):\n",
    "            result.append([self.highlighter(self.title_list[did], query),self.highlighter(self.doc_list[did], query), self.link_list[did], score])\n",
    "        return result\n",
    "    \n",
    "    def highlighter(self, doc, query):\n",
    "        \"\"\"用query对doc进行HTML高亮\n",
    "        \n",
    "        Args:\n",
    "            doc:需要高亮的文档\n",
    "            word:要进行高亮的关键词(查询)\n",
    "            \n",
    "        Returns:\n",
    "            返回对关键词(查询)进行高亮的文档\n",
    "        \"\"\"\n",
    "        n = 0\n",
    "        #生成要进行高亮的关键词串集合\n",
    "        word_set = set()\n",
    "        query = query.lower()\n",
    "        query_parts = self.parse_query(query)\n",
    "        for query_part in query_parts:\n",
    "            if query_part[0] == 'e' or query_part[0] == 'f':\n",
    "                word_set.add(query_part[1:])\n",
    "                if len(query_part[1:]) > n:\n",
    "                    n = len(query_part[1:])\n",
    "            elif query_part[0] == 'c':\n",
    "                if len(query_part[1:]) > 1:\n",
    "                    for word in jieba.cut(query_part[1:]):\n",
    "                        word_set.add(word)\n",
    "                        if len(word) > n:\n",
    "                            n = len(word)\n",
    "        \n",
    "        #遍历文档替换高亮关键词串\n",
    "        doc_low = doc.lower()\n",
    "        i = 0\n",
    "        result = []\n",
    "        side_len = 30 #设置上下文保留的宽度\n",
    "        last_end = 0 #上一个命中关键词的末位位置\n",
    "        limit_seg = 5 #保留命中关键词片段的条数\n",
    "        seg_count = 0\n",
    "        while True:\n",
    "            end_idx = i + n\n",
    "            if end_idx > len(doc_low):\n",
    "                end_idx = len(doc_low)\n",
    "            for j in range(end_idx, i, -1):\n",
    "                if doc_low[i:j] in word_set:\n",
    "                    break\n",
    "            if doc_low[i:j] in word_set:\n",
    "                #追加从上一个关键词末位到当前关键词开头的周围上下文\n",
    "                if i - last_end > 2 * side_len:\n",
    "                    result.append(doc[last_end:last_end+side_len])\n",
    "                    result.append('...')\n",
    "                    result.append(doc[i-side_len:i])\n",
    "                else:\n",
    "                    result.append(doc[last_end:i])\n",
    "                #追加高亮处理的关键词串\n",
    "                result.append(\n",
    "                    '<span style=\"color:red\">{}</span>'.format(doc[i:j]))\n",
    "                last_end = j\n",
    "                seg_count += 1\n",
    "                if seg_count > limit_seg:\n",
    "                    break\n",
    "#             else:\n",
    "#                 result.append(doc_low[i:j])\n",
    "            i = j\n",
    "            if i == len(doc_low):\n",
    "                right_idx = last_end + side_len\n",
    "                if right_idx > len(doc):\n",
    "                    result.append(doc[last_end:])\n",
    "                else:\n",
    "                    result.append(doc[last_end:right_idx])\n",
    "                    result.append('...')\n",
    "                break\n",
    "                \n",
    "        return ''.join(result)\n",
    "    \n",
    "    def conv_query(self, query):\n",
    "        \"\"\"将用户的查询转换成用eval可运行、返回结果ID集合的代码段\n",
    "        \n",
    "        Args:\n",
    "            query:待转换的原始查询字符串\n",
    "        \n",
    "        Returns:\n",
    "            提取出的关键词集，以及转换完成可通过eval执行返回ID集合的代码段字符串\n",
    "        \"\"\"\n",
    "        query_new_parts = []\n",
    "        keywords = []\n",
    "        all_parts = list(self.parse_query(query))\n",
    "        idx = 0\n",
    "        cache = '' #缓存变量，用于回收分词过程被切开的短语片段\n",
    "        count_parts = len(all_parts)\n",
    "        while idx < count_parts:\n",
    "            if all_parts[idx][1:] == '(' or all_parts[idx][1:] == ')':\n",
    "                query_new_parts.append(all_parts[idx][1:])\n",
    "            elif all_parts[idx][1:] == ' ':\n",
    "                query_new_parts.append(' ')\n",
    "            elif all_parts[idx][1:] in ('and', 'AND', '+'):\n",
    "                query_new_parts.append('&')\n",
    "            elif all_parts[idx][1:] in ('or', 'OR'):\n",
    "                query_new_parts.append('|')\n",
    "            elif all_parts[idx][1:] in ('not', 'NOT', '-'):\n",
    "                query_new_parts.append('-')\n",
    "            elif (idx + 1 < count_parts #对连续的内容分段结果集合中间加”&“运算符\n",
    "                  and all_parts[idx+1][1:] not in (' ', ')')): \n",
    "                query_new_parts.append(\"{} & \".format(self.conv_part(all_parts[idx])))\n",
    "                keywords += self.extract_keywords_part(all_parts[idx])\n",
    "            elif (idx + 2 < count_parts #处理词间、词与符号间空格的情况\n",
    "                  and all_parts[idx+1][1:] == \" \" \n",
    "                  and all_parts[idx+2][1:] not in (')', 'and', 'AND', '+', 'or', 'OR', 'NOT', 'not', '+', '-', ' ')): \n",
    "                query_new_parts.append(\"{} & \".format(self.conv_part(all_parts[idx])))\n",
    "                keywords += self.extract_keywords_part(all_parts[idx])\n",
    "                idx += 2\n",
    "                continue\n",
    "            else:\n",
    "                query_new_parts.append(self.conv_part(cache + all_parts[idx]))\n",
    "                keywords += self.extract_keywords_part(cache + all_parts[idx])\n",
    "                cache = '' #合并完成清空缓存\n",
    "            idx += 1\n",
    "        query_new = ''.join(query_new_parts)\n",
    "        return keywords, query_new\n",
    "    \n",
    "    def extract_keywords_part(self, part):\n",
    "        \"\"\"从带有类别标记的解析结果段提取关键词列表\n",
    "        \n",
    "        Args:\n",
    "            part:带有类别标记的解析结果段\n",
    "            \n",
    "        Results:\n",
    "            提取出的关键词列表\n",
    "        \"\"\"\n",
    "        flag = part[0]\n",
    "        if flag == 'e':\n",
    "            return [part[1:]]\n",
    "        elif flag == 'c':\n",
    "            return list(jieba.cut(part[1:]))\n",
    "        elif flag == 'f':\n",
    "            return [part[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_searcher = LuoIndex('luojisiwei.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "1.(0.011679320260501789) <a href=\"https://www.ljsw.io/knowl/article/Rk.html\" target=\"_blank\">第69期丨迷茫时代的明白人</a><br/><p>首播于2014年4月25日\n",
       "\n",
       "本期看点\n",
       "\n",
       "\n",
       "回到具体的历史情境\n",
       "<span style=\"color:red\">李鸿章</span>的一生\n",
       "<span style=\"color:red\">李鸿章</span>PK左宗棠：不沉浸于过往恩怨\n",
       "<span style=\"color:red\">李鸿章</span>PK张之洞：不跟历史叫板，不跟未来较劲\n",
       "<span style=\"color:red\">李鸿章</span>PK翁同龢：不被人际关系绑定\n",
       "看住当下，做现在该做的事情\n",
       "把心放在现在、今天、此刻\n",
       "\n",
       "\n",
       "本期荐书\n",
       "\n",
       "《<span style=\"color:red\">李鸿章</span>政改笔记》雪珥《<span style=\"color:red\">李鸿章</span></p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "2.(0.005348125226976596) <a href=\"https://www.ljsw.io/knowl/article/F4.html\" target=\"_blank\">第709期 | <span style=\"color:red\">曾国藩</span>是大清的功臣还是罪人？</a><br/><p>策划人：李子旸和你一起终身学习，这里是罗辑思维。最近看了唐浩明先生写的一本书《静远楼读史》。其中谈到<span style=\"color:red\">曾国藩</span>的一个问题，视角很独特，从来没有这么想过，今天和您聊聊。这个话题是：<span style=\"color:red\">曾国藩</span>是大清朝的功臣还是罪人？乍看起来，这问题问得好没道理。要不是<span style=\"color:red\">曾国藩</span>，太平天国可能就把大清朝一锅端了。哪里来后面的洋务运动、同光中兴？<span style=\"color:red\">曾国藩</span>是1872年死的。大清朝在他打的底子上又多活了将近40年啊。...是最近发现了什么新史料，揭开了什么前所未知的内幕。这都没有。<span style=\"color:red\">曾国藩</span>对大清朝的功也好，罪也罢，都是他组建湘军、打败太平天国这同一件事的结果。下面我们就来看看，<span style=\"color:red\">曾国藩</span></p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "3.(0.004224156296021859) <a href=\"https://www.ljsw.io/knowl/article/86.html\" target=\"_blank\">第419期 | <span style=\"color:red\">曾国藩</span>的算盘</a><br/><p>策划人：李子旸和你一起终身学习，这里是罗辑思维。一提到<span style=\"color:red\">曾国藩</span>，大家就会想到一句话：“结硬寨，打呆仗”，意思是说<span style=\"color:red\">曾国藩</span>打败太平天国的诀窍就是稳扎稳打，步步为营，不取巧，不投机，始...于“结硬寨，打呆仗”，不怕耗时间的，一般都是防守方，很少见到<span style=\"color:red\">曾国藩</span>这种进攻一方却要去“结硬寨，打呆仗”的。那么，<span style=\"color:red\">曾国藩</span>为什么会采用这种怪异的战法呢？说到战斗力和战斗意志，太平军其实在湘军之上。在几次硬碰硬的对决中，湘军都是惨败，以至于<span style=\"color:red\">曾国藩</span>屡次要自杀。但湘军在一个方面明显优于太平军。实际上，说太平天...，说：“一月不破城，必成瓦解之势。”正因为粮饷极为重要，所以<span style=\"color:red\">曾国藩</span></p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "4.(0.003270841216914237) <a href=\"https://www.ljsw.io/knowl/article/xq.html\" target=\"_blank\">第3期丨中日贸易，如何爱国</a><br/><p>快到年底了，有一则财经新闻很多人都知道。说日本经济因为中日之...国，这实在是一个一百多年来都打不清的官司。我们就说一个人吧，<span style=\"color:red\">李鸿章</span>。他卖国已经卖出品牌来了，后来西方列强找清政府签合约，都得叫<span style=\"color:red\">李鸿章</span>来。<span style=\"color:red\">李鸿章</span>不来这边人都不大放心。\n",
       "最典型的是《马关条约》了，日本人点名，你必须<span style=\"color:red\">李鸿章</span>到马关来签。但是<span style=\"color:red\">李鸿章</span>他气死了，所以在《马关条约》之后，1896年他有一次环球航行...爱国是一种正常情绪，你说伟大，我觉得是一种过奖之词。\n",
       "你搁到<span style=\"color:red\">李鸿章</span></p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "5.(0.0032622651141537345) <a href=\"https://www.ljsw.io/knowl/article/Ra.html\" target=\"_blank\">第82期丨<span style=\"color:red\">曾国藩</span>成功学</a><br/><p>首播于2014年7月24日\n",
       "\n",
       "本期看点\n",
       "\n",
       "\n",
       "笨蛋”<span style=\"color:red\">曾国藩</span>\n",
       "青年时代，用自己的道德标准要求别人\n",
       "失落回乡，心性大变\n",
       "第二次复出：放下身段，长袖善舞\n",
       "<span style=\"color:red\">曾国藩</span>的正面和侧面\n",
       "“不给他人难堪”背后的逻辑\n",
       "内圣外王的人生境界\n",
       "\n",
       "\n",
       "本期策划\n",
       "\n",
       "刘学\n",
       "\n",
       "本期荐书\n",
       "\n",
       "《<span style=\"color:red\">曾国藩</span>的正面与侧面》张宏杰</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "6.(0.002424804684653415) <a href=\"https://www.ljsw.io/knowl/article/FO.html\" target=\"_blank\">第725期 | 中国会成为科技大国吗？</a><br/><p>策划人：李子旸和你一起终身学习，这里是罗辑思维。有一个话题，...会耍什么花招，清政府心里是完全没底的。事实上，就算是40年后<span style=\"color:red\">李鸿章</span>修建的津沪线，也是小心提防洋人的。比如，建设津沪电报线的时候，就特意选择了一家丹麦的电报公司。<span style=\"color:red\">李鸿章</span>在奏折里说：丹麦是小国，小国商人比较听话，容易为我所用。这哪...政。北京和慈禧太后所在的西安，都不是信息中心。这下你就理解了<span style=\"color:red\">李鸿章</span>的苦心，为什么要把这段电报线的北端设在天津而不是北京了。这个...了电报的速度，又兼顾了原来的信息系统和政治系统。当然了，不管<span style=\"color:red\">李鸿章</span>这一代人对电报有多警惕，有多少聪明的防范措施，新技术的扩散还...现之后，奏折这玩意变了。没有电报之前，大臣奏事，都是写奏折。<span style=\"color:red\">曾国藩</span><span style=\"color:red\">李鸿章</span></p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "7.(0.0019041242675424614) <a href=\"https://www.ljsw.io/weixin/2015-07-10/9l.html\" target=\"_blank\">#932 历史 | 《慈禧全传》：一部小说的后遗症</a><br/><p>今天我向大家推荐的，是高阳先生的巨著《慈禧全传》。这是罗辑思...就知道怎么抒发；读过苏东坡，“败”的时候就不至于太狼狈；读过<span style=\"color:red\">李鸿章</span>，“利”的时候就知道怎么乘胜；读过<span style=\"color:red\">曾国藩</span>，“钝”的时候就知道怎么挺住。总之，读得越多，我们大脑中预装...官僚士大夫的思维情境里，移步换景。等再出来时，慈禧、恭亲王、<span style=\"color:red\">曾国藩</span>、<span style=\"color:red\">李鸿章</span>、袁世凯……这些人的思维方式已经悄悄内化在我们的生命中了。这...些道学先生的存在，就无法理解慈禧是一个真正的改革家；你不了解<span style=\"color:red\">曾国藩</span>所处的微妙位置，就无法想通圣人般的他为什么在攻灭太平天国后，...</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "8.(0.001854780085270816) <a href=\"https://www.ljsw.io/weixin/2018-05-22/ADj.html\" target=\"_blank\">#1977 合伙人 | 罗胖60秒：怎么找事业的合伙人？</a><br/><p>今天是罗胖陪伴你的第 1979 天1. <span style=\"color:red\">曾国藩</span>有一句话，说“与多疑人共事，事必不成。与好利人共事，己必受累.... 你想，谁不多疑啊，谁不喜欢利益啊，这是人的本能啊。所以，<span style=\"color:red\">曾国藩</span>到底在说什么？他是在说选择事业伙伴的原则。3. 通常我们说服...找事业合伙人，那用这两个手段，达不成目标，反而埋下隐患。就是<span style=\"color:red\">曾国藩</span>讲的，要么事情干不成，要么自己受拖累。4. 为什么？答案就藏在<span style=\"color:red\">曾国藩</span>说的那个词里，这是共事，是长期的事业共同体，不是短期达成目标...</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "9.(0.0018530175034087472) <a href=\"https://www.ljsw.io/knowl/article/9j.html\" target=\"_blank\">第737期 | 你面对的是单纯问题吗？</a><br/><p>和你一起终身学习，这里是罗辑思维。上周，我们聊了一个话题，叫...样子。这篇文章，我看到这里的时候，想起了一个人，那就是晚清的<span style=\"color:red\">李鸿章</span>。他自己说自己是大清的“裱糊匠”。原话大概是这样的，<span style=\"color:red\">李鸿章</span>说：“我办了一辈子的事，练兵，办海军，其实都是纸糊的老虎。从...把纸全部扯掉，最后的结果只能是不可收拾。”你看，在这段话里，<span style=\"color:red\">李鸿章</span>的自我定位，就是一个处理“棘手问题”的人。其实何止是他，几乎...后总结出来的啊。只要身在其中的人，每一步都极为艰难，都是在像<span style=\"color:red\">李鸿章</span>一样当裱糊匠而已。但是还有一种人，就是做事的旁观者。就很容易...</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "10.(0.0016025641025641025) <a href=\"https://www.ljsw.io/knowl/article/zm.html\" target=\"_blank\">第610期 | 一本穿针引线的书</a><br/><p>和你一起终身学习，这里是罗辑思维。今天，得到App首发一本书...丰富的维度，更丰富的细节，来帮你还原。我举个例子，比如，提到<span style=\"color:red\">曾国藩</span>这个人，你会想起什么？战略家，中兴功臣，鸡汤文作者，将领，或...杀农民起义的刽子手。不管是什么印象，这都是符号，你还是没能把<span style=\"color:red\">曾国藩</span>这个人还原到他生存的那个真实世界里。但是，如果你读《谭伯牛说晚清史》，你会在里面发现<span style=\"color:red\">曾国藩</span>说的一句俏皮话， 这是他说书法的，他说啊，写字要像“少妇谋杀亲夫”，这话什么意思呢？<span style=\"color:red\">曾国藩</span>的解释是：既美且狠。所以像少妇谋杀亲夫。怎么样？读到了这个细节，<span style=\"color:red\">曾国藩</span>的形象是不是立体丰满起来？他平时打仗，做官，做学问，但是，平...时候，就和我们身边的一个爱说俏皮话的同事没有什么区别。这样的<span style=\"color:red\">曾国藩</span></p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "11.(0.0014391450482914654) <a href=\"https://www.ljsw.io/weixin/2015-11-20/yz.html\" target=\"_blank\">#1065 格局 | 局中人与破网者</a><br/><p>罗胖的推荐：今天推荐的这本书《顽疾》，作者是著名的历史学者张...他是如何釜底抽薪，编织了这样一张良性的网？《顽疾》会告诉你。<span style=\"color:red\">曾国藩</span>怎么破出一张恶性的网？<span style=\"color:red\">曾国藩</span>，立志学做圣人，成为一个道德上的完人。但他却不想做海瑞那样一...，但一辈子却没做成什么大事的人。于是在现存记录中，我们找不到<span style=\"color:red\">曾国藩</span>把任何一分公款装入自己腰包的记录，甚至他当京官都无力救济自己...小金库，也按照官场惯例打点上级，甚至跟别的官员一起大吃大喝。<span style=\"color:red\">曾国藩</span>取一尘不染之实，却竭尽全力避免一清如水之名。他是怎样过着分裂...史写作者，他之前写历史的几本著作：《大明王朝的七张面孔》、《<span style=\"color:red\">曾国藩</span>的正面与侧面》、《饥饿的盛世》，罗胖在视频节目里多次说起。这...</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "12.(0.0012452431896073368) <a href=\"https://www.ljsw.io/knowl/article/BO.html\" target=\"_blank\">第411期 | 重新理解中国·之四</a><br/><p>和你一起终身学习，这里是罗辑思维。「得到」App刚刚更新了版...着头皮起用汉族地方官员，让他们想辙看能不能摆平太平天国。于是<span style=\"color:red\">曾国藩</span>、左宗棠、<span style=\"color:red\">李鸿章</span>、胡林翼等等一批晚清中兴名臣，就在这个过程中走上了历史舞台。...能够最终击败太平天国，就是靠的这个力量。有意思的问题就来了，<span style=\"color:red\">曾国藩</span>湘军、<span style=\"color:red\">李鸿章</span>淮军等等这些军队，规模可是不小，军饷发得也相当高。那意味着，...</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "13.(0.0009676847487769534) <a href=\"https://www.ljsw.io/weixin/2017-07-18/Qy.html\" target=\"_blank\">#1671 解释 | 罗胖60秒：《罗辑思维》300期，一键...</a><br/><p>借用 武志红 老师的一句话，“真正的好知识，就是穿过了身体的...的行动上，我的努力也就值了。埃隆·马斯克︱爱迪生︱爱因斯坦︱<span style=\"color:red\">曾国藩</span>︱查理·芒格︱崇祯︱慈禧︱达尔文︱大卫与歌利亚︱杜月笙︱福特...卢斯︱忽必烈︱胡适︱嘉靖︱蒋介石︱凯撒︱凯文·凯利︱康有为︱<span style=\"color:red\">李鸿章</span>︱里根︱林肯︱林则徐︱刘邦︱卢梭︱罗斯福︱洛克菲勒︱麦克阿瑟...创的世界史︱白银帝国︱百岁人生︱伯罗奔尼撒战争︱财富的帝国︱<span style=\"color:red\">曾国藩</span>的正面与侧面︱出版人︱创造︱春秋大义︱慈禧全传︱大法官说了算...</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "14.(0.0008185158949603351) <a href=\"https://www.ljsw.io/knowl/article/RO.html\" target=\"_blank\">第99期丨成大事者不纠结</a><br/><p>首播于2014年11月21日\n",
       "\n",
       "本期看点\n",
       "\n",
       "\n",
       "太平天国是如何滚雪球的\n",
       "<span style=\"color:red\">曾国藩</span>复出后的困境\n",
       "六字心法打天下：“结硬寨、打呆仗”\n",
       "处理人际关...</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "15.(0.0007138437050809081) <a href=\"https://www.ljsw.io/knowl/article/FU.html\" target=\"_blank\">第710期 | 为什么要警惕“代偿效应”？</a><br/><p>策划人：裘德和你一起终身学习，这里是罗辑思维。昨天我们通过<span style=\"color:red\">曾国藩</span>这个例子，试图说明一个效应：“代偿效应”的潜在危害性。什么是...，这场战争等发现的时候，就真的危险了。你看，这和我们昨天讲的<span style=\"color:red\">曾国藩</span>和大清朝的关系是不是有点像？靠一个代偿性的功能暂时度过危机，...现象，其实就是这两种不同游戏规则之间的矛盾。回到我们昨天说的<span style=\"color:red\">曾国藩</span>那个例子。大清朝廷本质上是在追求无限游戏，一个王朝，当然想千...死。这就转换了游戏的性质，变成了有限游戏。所以，它就只好启用<span style=\"color:red\">曾国藩</span>这样的人，也就是它的代偿机制。拖到最后万病齐发，不可救药。这...</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "query = '李鸿章 OR 曾国藩'\n",
    "result = news_searcher.search(query)\n",
    "count = 1\n",
    "if result:\n",
    "    for title, doc, url, score in result[:15]:\n",
    "        display(HTML('{}.({}) <a href=\"{}\" target=\"_blank\">{}</a><br/><p>{}</p>'.format(count, score, url, title, doc)))\n",
    "        count += 1\n",
    "else:\n",
    "    print('No result.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}