3.per_article_pretain_vec_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "\n",
    "from gensim.models import word2vec\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.preprocessing import LabelEncoder,OneHotEncoder\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential,Model\n",
    "from keras.layers import Dense, Embedding, SpatialDropout1D, Dropout, Activation, Input, LSTM, Bidirectional\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.utils import to_categorical\n",
    "import re\n",
    "\n",
    "#time\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "\n",
    "import jieba\n",
    "import jieba.analyse\n",
    "jieba.set_dictionary('dict.idkrsi.txt')     # 改預設字典\n",
    "jieba.analyse.set_stop_words(\"stopword.goatwang.kang.txt\") #指定stopwords字典\n",
    "\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "\n",
    "\n",
    "# get data\n",
    "# ! conda install pandas-datareader s\n",
    "#import pandas_datareader as pdr\n",
    "\n",
    "# visual\n",
    "# ! pip install mpl-finance\n",
    "#import matplotlib.pyplot as plt\n",
    "#import mpl_finance as mpf\n",
    "\n",
    "#import seaborn as sns\n",
    "\n",
    "\n",
    "# https://github.com/mrjbq7/ta-lib\n",
    "# ! pip install ta-lib\n",
    "#import talib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/barry/miniconda3/envs/stockenv/lib/python3.6/site-packages/ipykernel_launcher.py:6: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>content</th>\n",
       "      <th>id</th>\n",
       "      <th>p_type</th>\n",
       "      <th>page_url</th>\n",
       "      <th>post_time</th>\n",
       "      <th>s_area_name</th>\n",
       "      <th>s_name</th>\n",
       "      <th>title</th>\n",
       "      <th>post_time2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>鉅亨網</td>\n",
       "      <td>0</td>\n",
       "      <td>鉅亨網編譯郭照青這一年，獲利真是難上難。&lt;BR&gt;根據CNBC分析報導，興業銀行的數據顯示，2...</td>\n",
       "      <td>1451580107540_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-01 00:06:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>淪落的2015 竟是78年來最難獲利的一年</td>\n",
       "      <td>2016-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bokolo</td>\n",
       "      <td>10</td>\n",
       "      <td>1.原文連結(必須檢附)： http://magazine.chinatimes.com/m...</td>\n",
       "      <td>1451580710262_PTT02R</td>\n",
       "      <td>bbs</td>\n",
       "      <td>http://www.ptt.cc/bbs/Stock/M.1451580509.A.722...</td>\n",
       "      <td>2016-01-01 00:48:25</td>\n",
       "      <td>Stock</td>\n",
       "      <td>Ptt</td>\n",
       "      <td>[新聞] 第三方支付 台灣普及關鍵年</td>\n",
       "      <td>2016-01-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   author  comment_count                                            content  \\\n",
       "0     鉅亨網              0  鉅亨網編譯郭照青這一年，獲利真是難上難。<BR>根據CNBC分析報導，興業銀行的數據顯示，2...   \n",
       "1  Bokolo             10  1.原文連結(必須檢附)： http://magazine.chinatimes.com/m...   \n",
       "\n",
       "                     id p_type  \\\n",
       "0     1451580107540_N01   news   \n",
       "1  1451580710262_PTT02R    bbs   \n",
       "\n",
       "                                            page_url           post_time  \\\n",
       "0  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-01 00:06:00   \n",
       "1  http://www.ptt.cc/bbs/Stock/M.1451580509.A.722... 2016-01-01 00:48:25   \n",
       "\n",
       "  s_area_name   s_name                  title  post_time2  \n",
       "0        重大要聞  yahoo股市  淪落的2015 竟是78年來最難獲利的一年  2016-01-01  \n",
       "1       Stock      Ptt     [新聞] 第三方支付 台灣普及關鍵年  2016-01-01  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bbs = pd.read_csv(\"bda2019_dataset/bbs2.csv\",encoding=\"utf-8\")\n",
    "df_forum = pd.read_csv(\"bda2019_dataset/forum2.csv\",encoding=\"utf-8\")\n",
    "df_news = pd.read_csv(\"bda2019_dataset/news2.csv\",encoding=\"utf-8\")\n",
    "df_news['comment_count']=0\n",
    "\n",
    "df_article = pd.concat([df_forum, df_bbs, df_news])  #三個合併\n",
    "del df_bbs, df_forum, df_news\n",
    "\n",
    "df_article['post_time'] = pd.to_datetime(df_article['post_time'])\n",
    "df_article['post_time2'] = df_article['post_time'].dt.date   # .dt.date用在dataframe  .date()用在一個 #只留日期\n",
    "#df_article['label'] = 'even'\n",
    "df_article['content'] = df_article['content'].astype(str).str.replace(',' , ' ').str.replace('\\n' , ' ').str.replace('\"' , ' ').str.replace(\"'\" , ' ')\n",
    "df_article['title'] = df_article['title'].astype(str).str.replace(',' , ' ').str.replace('\\n' , ' ').str.replace('\"' , ' ').str.replace(\"'\" , ' ')\n",
    "df_article = df_article.sort_values(by=['post_time']).reset_index(drop=True) # 用post_time排序 # 在重設index\n",
    "df_article.head(2)\n",
    "#df_article2 = df_article[['post_time2','title','content']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>證券代碼</th>\n",
       "      <th>年月日</th>\n",
       "      <th>開盤價(元)</th>\n",
       "      <th>最高價(元)</th>\n",
       "      <th>最低價(元)</th>\n",
       "      <th>收盤價(元)</th>\n",
       "      <th>成交量(千股)</th>\n",
       "      <th>成交值(千元)</th>\n",
       "      <th>成交筆數(筆)</th>\n",
       "      <th>流通在外股數(千股)</th>\n",
       "      <th>本益比-TSE</th>\n",
       "      <th>股價淨值比-TSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101 台泥</td>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>29.09</td>\n",
       "      <td>29.64</td>\n",
       "      <td>28.93</td>\n",
       "      <td>29.64</td>\n",
       "      <td>6121.0</td>\n",
       "      <td>213121.0</td>\n",
       "      <td>2939</td>\n",
       "      <td>3692175.0</td>\n",
       "      <td>20.56</td>\n",
       "      <td>1.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1102 亞泥</td>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>24.91</td>\n",
       "      <td>24.91</td>\n",
       "      <td>24.67</td>\n",
       "      <td>24.81</td>\n",
       "      <td>3347.0</td>\n",
       "      <td>88078.0</td>\n",
       "      <td>1227</td>\n",
       "      <td>3361447.0</td>\n",
       "      <td>31.75</td>\n",
       "      <td>0.73</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      證券代碼        年月日  開盤價(元)  最高價(元)  最低價(元)  收盤價(元)  成交量(千股)   成交值(千元)  \\\n",
       "0  1101 台泥 2016-12-30   29.09   29.64   28.93   29.64   6121.0  213121.0   \n",
       "1  1102 亞泥 2016-12-30   24.91   24.91   24.67   24.81   3347.0   88078.0   \n",
       "\n",
       "   成交筆數(筆)  流通在外股數(千股)  本益比-TSE  股價淨值比-TSE  \n",
       "0     2939   3692175.0    20.56       1.23  \n",
       "1     1227   3361447.0    31.75       0.73  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_TWSE2018 = pd.read_csv(\"bda2019_dataset/TWSE2018.csv\",encoding=\"utf-8\")\n",
    "df_TWSE2017 = pd.read_csv(\"bda2019_dataset/TWSE2017.csv\",encoding=\"utf-8\")\n",
    "df_TWSE2016 = pd.read_csv(\"bda2019_dataset/TWSE2016.csv\",encoding=\"utf-8\")\n",
    "\n",
    "df_TWSE = pd.concat([df_TWSE2016, df_TWSE2017, df_TWSE2018])  #三年合併\n",
    "del df_TWSE2016, df_TWSE2017, df_TWSE2018\n",
    "\n",
    "# ['開盤價(元)', '最高價(元)', '最低價(元)', '收盤價(元)', '成交量(千股)', '成交值(千元)', '成交筆數(筆)', '流通在外股數(千股)', '本益比-TSE', '股價淨值比-TSE']\n",
    "df_TWSE['證券代碼'] = df_TWSE['證券代碼'].astype(str)\n",
    "df_TWSE['年月日'] = pd.to_datetime(df_TWSE['年月日'])\n",
    "df_TWSE['開盤價(元)'] = df_TWSE['開盤價(元)'].str.replace(',' , '').astype('float64') # 1,000 to 1000 to float\n",
    "df_TWSE['最高價(元)'] = df_TWSE['最高價(元)'].str.replace(',' , '').astype('float64')\n",
    "df_TWSE['最低價(元)'] = df_TWSE['最低價(元)'].str.replace(',' , '').astype('float64')\n",
    "df_TWSE['收盤價(元)'] = df_TWSE['收盤價(元)'].str.replace(',' , '').astype('float64')\n",
    "df_TWSE['成交量(千股)'] = df_TWSE['成交量(千股)'].str.replace(',' , '').astype('float64')\n",
    "df_TWSE['成交值(千元)'] = df_TWSE['成交值(千元)'].str.replace(',' , '').astype('float64')\n",
    "df_TWSE['成交筆數(筆)'] = df_TWSE['成交筆數(筆)'].str.replace(',' , '').astype('int64')\n",
    "df_TWSE['流通在外股數(千股)'] = df_TWSE['流通在外股數(千股)'].str.replace(',' , '').astype('float64')\n",
    "df_TWSE['本益比-TSE'] = df_TWSE['本益比-TSE'].str.replace(',' , '').astype('float64')\n",
    "df_TWSE['股價淨值比-TSE'] = df_TWSE['股價淨值比-TSE'].astype('float64')\n",
    "df_TWSE.head(2) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 選那家股票\n",
    "#company_name = '國巨'\n",
    "company_name = '奇力新'\n",
    "\n",
    "\n",
    "# 文章包含那家字\n",
    "#company_words = '被動元件|積層陶瓷電容|MLCC|電感|晶片電阻|車用電子|凱美|同欣電|大毅|君耀|普斯|國巨'\n",
    "company_words = '被動元件|積層陶瓷電容|MLCC|電感|晶片電阻|車用電子|飛磁|旺詮|美磊|美桀|向華科技|奇力新'\n",
    "\n",
    "# 漲跌幾%\n",
    "PA = 0.05\n",
    "\n",
    "# even幾%\n",
    "PAE = 0.003"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>證券代碼</th>\n",
       "      <th>年月日</th>\n",
       "      <th>開盤價(元)</th>\n",
       "      <th>最高價(元)</th>\n",
       "      <th>最低價(元)</th>\n",
       "      <th>收盤價(元)</th>\n",
       "      <th>成交量(千股)</th>\n",
       "      <th>成交值(千元)</th>\n",
       "      <th>成交筆數(筆)</th>\n",
       "      <th>流通在外股數(千股)</th>\n",
       "      <th>本益比-TSE</th>\n",
       "      <th>股價淨值比-TSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-04</td>\n",
       "      <td>80.44</td>\n",
       "      <td>81.74</td>\n",
       "      <td>78.18</td>\n",
       "      <td>80.20</td>\n",
       "      <td>4389.0</td>\n",
       "      <td>297206.0</td>\n",
       "      <td>2710</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>15.89</td>\n",
       "      <td>2.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>80.55</td>\n",
       "      <td>85.06</td>\n",
       "      <td>79.25</td>\n",
       "      <td>85.06</td>\n",
       "      <td>6701.0</td>\n",
       "      <td>468760.0</td>\n",
       "      <td>4200</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>16.85</td>\n",
       "      <td>2.46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       證券代碼        年月日  開盤價(元)  最高價(元)  最低價(元)  收盤價(元)  成交量(千股)   成交值(千元)  \\\n",
       "0  2456 奇力新 2016-01-04   80.44   81.74   78.18   80.20   4389.0  297206.0   \n",
       "1  2456 奇力新 2016-01-05   80.55   85.06   79.25   85.06   6701.0  468760.0   \n",
       "\n",
       "   成交筆數(筆)  流通在外股數(千股)  本益比-TSE  股價淨值比-TSE  \n",
       "0     2710    126969.0    15.89       2.32  \n",
       "1     4200    126969.0    16.85       2.46  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 用日期排序 再把index重排\n",
    "#2327 \n",
    "#df_trend = df_TWSE[df_TWSE['證券代碼'].str.contains('國巨')].sort_values(by=['年月日']).reset_index(drop=True)\n",
    "#2456 \n",
    "#df_trend = df_TWSE[df_TWSE['證券代碼'].str.contains('奇力新')].sort_values(by=['年月日']).reset_index(drop=True)\n",
    "#2478 \n",
    "#df_trend = df_TWSE[df_TWSE['證券代碼'].str.contains('大毅')].sort_values(by=['年月日']).reset_index(drop=True)\n",
    "#6271 \n",
    "#df_trend = df_TWSE[df_TWSE['證券代碼'].str.contains('同欣電')].sort_values(by=['年月日']).reset_index(drop=True)\n",
    "\n",
    "df_trend = df_TWSE[df_TWSE['證券代碼'].str.contains(company_name)].sort_values(by=['年月日']).reset_index(drop=True)\n",
    "del df_TWSE\n",
    "df_trend.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>證券代碼</th>\n",
       "      <th>年月日</th>\n",
       "      <th>開盤價(元)</th>\n",
       "      <th>最高價(元)</th>\n",
       "      <th>最低價(元)</th>\n",
       "      <th>收盤價(元)</th>\n",
       "      <th>成交量(千股)</th>\n",
       "      <th>成交值(千元)</th>\n",
       "      <th>成交筆數(筆)</th>\n",
       "      <th>流通在外股數(千股)</th>\n",
       "      <th>本益比-TSE</th>\n",
       "      <th>股價淨值比-TSE</th>\n",
       "      <th>fluctuation</th>\n",
       "      <th>tag</th>\n",
       "      <th>closeshift</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-04</td>\n",
       "      <td>80.44</td>\n",
       "      <td>81.74</td>\n",
       "      <td>78.18</td>\n",
       "      <td>80.20</td>\n",
       "      <td>4389.0</td>\n",
       "      <td>297206.0</td>\n",
       "      <td>2710</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>15.89</td>\n",
       "      <td>2.32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>80.55</td>\n",
       "      <td>85.06</td>\n",
       "      <td>79.25</td>\n",
       "      <td>85.06</td>\n",
       "      <td>6701.0</td>\n",
       "      <td>468760.0</td>\n",
       "      <td>4200</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>16.85</td>\n",
       "      <td>2.46</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       證券代碼        年月日  開盤價(元)  最高價(元)  最低價(元)  收盤價(元)  成交量(千股)   成交值(千元)  \\\n",
       "0  2456 奇力新 2016-01-04   80.44   81.74   78.18   80.20   4389.0  297206.0   \n",
       "1  2456 奇力新 2016-01-05   80.55   85.06   79.25   85.06   6701.0  468760.0   \n",
       "\n",
       "   成交筆數(筆)  流通在外股數(千股)  本益比-TSE  股價淨值比-TSE  fluctuation tag  closeshift  \n",
       "0     2710    126969.0    15.89       2.32          0.0  --         0.0  \n",
       "1     4200    126969.0    16.85       2.46          0.0  --         0.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##增欄位:fluctuation幅度 tag漲跌平\n",
    "df_trend['fluctuation'] = 0.0\n",
    "df_trend['tag']='--'\n",
    "df_trend['closeshift'] = 0.0\n",
    "df_trend.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ##增欄位:fluctuation幅度 tag漲跌平\n",
    "# df_trend['fluctuation'] = 0.0\n",
    "# df_trend['tag']='--'\n",
    "\n",
    "# ###計算漲跌\n",
    "# for index, row in df_trend.iterrows():\n",
    "#     try:\n",
    "#         margin =(float(df_trend.loc[index,'收盤價(元)']) - float(df_trend.loc[index-1,'收盤價(元)']) )/ float(df_trend.loc[index-1,'收盤價(元)'])\n",
    "#         df_trend.loc[index,'fluctuation']=margin\n",
    "#         if margin >=0.03:\n",
    "#             df_trend.loc[index,'tag']='up'\n",
    "#         elif margin <= -0.03:\n",
    "#             df_trend.loc[index,'tag']='down'\n",
    "#         else:\n",
    "#             df_trend.loc[index,'tag']='even'\n",
    "#     except:\n",
    "#         continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>證券代碼</th>\n",
       "      <th>年月日</th>\n",
       "      <th>開盤價(元)</th>\n",
       "      <th>最高價(元)</th>\n",
       "      <th>最低價(元)</th>\n",
       "      <th>收盤價(元)</th>\n",
       "      <th>成交量(千股)</th>\n",
       "      <th>成交值(千元)</th>\n",
       "      <th>成交筆數(筆)</th>\n",
       "      <th>流通在外股數(千股)</th>\n",
       "      <th>本益比-TSE</th>\n",
       "      <th>股價淨值比-TSE</th>\n",
       "      <th>fluctuation</th>\n",
       "      <th>tag</th>\n",
       "      <th>closeshift</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-04</td>\n",
       "      <td>80.44</td>\n",
       "      <td>81.74</td>\n",
       "      <td>78.18</td>\n",
       "      <td>80.20</td>\n",
       "      <td>4389.0</td>\n",
       "      <td>297206.0</td>\n",
       "      <td>2710</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>15.89</td>\n",
       "      <td>2.32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>80.55</td>\n",
       "      <td>85.06</td>\n",
       "      <td>79.25</td>\n",
       "      <td>85.06</td>\n",
       "      <td>6701.0</td>\n",
       "      <td>468760.0</td>\n",
       "      <td>4200</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>16.85</td>\n",
       "      <td>2.46</td>\n",
       "      <td>0.0</td>\n",
       "      <td>--</td>\n",
       "      <td>80.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       證券代碼        年月日  開盤價(元)  最高價(元)  最低價(元)  收盤價(元)  成交量(千股)   成交值(千元)  \\\n",
       "0  2456 奇力新 2016-01-04   80.44   81.74   78.18   80.20   4389.0  297206.0   \n",
       "1  2456 奇力新 2016-01-05   80.55   85.06   79.25   85.06   6701.0  468760.0   \n",
       "\n",
       "   成交筆數(筆)  流通在外股數(千股)  本益比-TSE  股價淨值比-TSE  fluctuation tag  closeshift  \n",
       "0     2710    126969.0    15.89       2.32          0.0  --         NaN  \n",
       "1     4200    126969.0    16.85       2.46          0.0  --        80.2  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trend['closeshift'] = df_trend['收盤價(元)'].shift(periods=1)#.fillna(value=0.0, inplace=True)\n",
    "#df_trend['closeshift'].fillna(value= 0.0, inplace=True)\n",
    "df_trend.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>證券代碼</th>\n",
       "      <th>年月日</th>\n",
       "      <th>開盤價(元)</th>\n",
       "      <th>最高價(元)</th>\n",
       "      <th>最低價(元)</th>\n",
       "      <th>收盤價(元)</th>\n",
       "      <th>成交量(千股)</th>\n",
       "      <th>成交值(千元)</th>\n",
       "      <th>成交筆數(筆)</th>\n",
       "      <th>流通在外股數(千股)</th>\n",
       "      <th>本益比-TSE</th>\n",
       "      <th>股價淨值比-TSE</th>\n",
       "      <th>fluctuation</th>\n",
       "      <th>tag</th>\n",
       "      <th>closeshift</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-04</td>\n",
       "      <td>80.44</td>\n",
       "      <td>81.74</td>\n",
       "      <td>78.18</td>\n",
       "      <td>80.20</td>\n",
       "      <td>4389.0</td>\n",
       "      <td>297206.0</td>\n",
       "      <td>2710</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>15.89</td>\n",
       "      <td>2.32</td>\n",
       "      <td>NaN</td>\n",
       "      <td>--</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>80.55</td>\n",
       "      <td>85.06</td>\n",
       "      <td>79.25</td>\n",
       "      <td>85.06</td>\n",
       "      <td>6701.0</td>\n",
       "      <td>468760.0</td>\n",
       "      <td>4200</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>16.85</td>\n",
       "      <td>2.46</td>\n",
       "      <td>0.060599</td>\n",
       "      <td>--</td>\n",
       "      <td>80.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       證券代碼        年月日  開盤價(元)  最高價(元)  最低價(元)  收盤價(元)  成交量(千股)   成交值(千元)  \\\n",
       "0  2456 奇力新 2016-01-04   80.44   81.74   78.18   80.20   4389.0  297206.0   \n",
       "1  2456 奇力新 2016-01-05   80.55   85.06   79.25   85.06   6701.0  468760.0   \n",
       "\n",
       "   成交筆數(筆)  流通在外股數(千股)  本益比-TSE  股價淨值比-TSE  fluctuation tag  closeshift  \n",
       "0     2710    126969.0    15.89       2.32          NaN  --         NaN  \n",
       "1     4200    126969.0    16.85       2.46     0.060599  --        80.2  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trend['fluctuation'] = (df_trend['收盤價(元)'] - df_trend['closeshift']) / df_trend['closeshift']\n",
    "df_trend.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>證券代碼</th>\n",
       "      <th>年月日</th>\n",
       "      <th>開盤價(元)</th>\n",
       "      <th>最高價(元)</th>\n",
       "      <th>最低價(元)</th>\n",
       "      <th>收盤價(元)</th>\n",
       "      <th>成交量(千股)</th>\n",
       "      <th>成交值(千元)</th>\n",
       "      <th>成交筆數(筆)</th>\n",
       "      <th>流通在外股數(千股)</th>\n",
       "      <th>本益比-TSE</th>\n",
       "      <th>股價淨值比-TSE</th>\n",
       "      <th>fluctuation</th>\n",
       "      <th>tag</th>\n",
       "      <th>closeshift</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-04</td>\n",
       "      <td>80.44</td>\n",
       "      <td>81.74</td>\n",
       "      <td>78.18</td>\n",
       "      <td>80.20</td>\n",
       "      <td>4389.0</td>\n",
       "      <td>297206.0</td>\n",
       "      <td>2710</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>15.89</td>\n",
       "      <td>2.32</td>\n",
       "      <td>NaN</td>\n",
       "      <td>--</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2456 奇力新</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>80.55</td>\n",
       "      <td>85.06</td>\n",
       "      <td>79.25</td>\n",
       "      <td>85.06</td>\n",
       "      <td>6701.0</td>\n",
       "      <td>468760.0</td>\n",
       "      <td>4200</td>\n",
       "      <td>126969.0</td>\n",
       "      <td>16.85</td>\n",
       "      <td>2.46</td>\n",
       "      <td>0.060599</td>\n",
       "      <td>up</td>\n",
       "      <td>80.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       證券代碼        年月日  開盤價(元)  最高價(元)  最低價(元)  收盤價(元)  成交量(千股)   成交值(千元)  \\\n",
       "0  2456 奇力新 2016-01-04   80.44   81.74   78.18   80.20   4389.0  297206.0   \n",
       "1  2456 奇力新 2016-01-05   80.55   85.06   79.25   85.06   6701.0  468760.0   \n",
       "\n",
       "   成交筆數(筆)  流通在外股數(千股)  本益比-TSE  股價淨值比-TSE  fluctuation tag  closeshift  \n",
       "0     2710    126969.0    15.89       2.32          NaN  --         NaN  \n",
       "1     4200    126969.0    16.85       2.46     0.060599  up        80.2  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trend.loc[df_trend['fluctuation'] >=  PA, 'tag'] = 'up'\n",
    "df_trend.loc[df_trend['fluctuation'] <= -PA, 'tag'] = 'down'\n",
    "df_trend.loc[(df_trend['fluctuation'] >= -PAE) & (df_trend['fluctuation'] <= PAE), 'tag'] = 'even'\n",
    "df_trend.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_trend[df_trend['tag']=='up'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_trend[df_trend['tag']=='down'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "90"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_trend[df_trend['tag']=='even'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10553\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>content</th>\n",
       "      <th>id</th>\n",
       "      <th>p_type</th>\n",
       "      <th>page_url</th>\n",
       "      <th>post_time</th>\n",
       "      <th>s_area_name</th>\n",
       "      <th>s_name</th>\n",
       "      <th>title</th>\n",
       "      <th>post_time2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>時報資訊</td>\n",
       "      <td>0</td>\n",
       "      <td>台股2015年下半年跌勢洶洶，全年跌幅達10.14％，但12檔具利多題材個股，股價多頭氣盛，...</td>\n",
       "      <td>1451618809219_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-01 10:59:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>《熱門族群》12檔話題當紅股，完美封關</td>\n",
       "      <td>2016-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>鉅亨網</td>\n",
       "      <td>0</td>\n",
       "      <td>鉅亨網記者王莞甯 台北台股2015年封關日最後一檔有神秘買盤進場，強拉加權指數上漲39點，元...</td>\n",
       "      <td>1451715108530_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-02 13:50:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>錢冠州：總統大選不再是大變數 元月行情伺機而動！</td>\n",
       "      <td>2016-01-02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   author  comment_count                                            content  \\\n",
       "30   時報資訊              0  台股2015年下半年跌勢洶洶，全年跌幅達10.14％，但12檔具利多題材個股，股價多頭氣盛，...   \n",
       "97    鉅亨網              0  鉅亨網記者王莞甯 台北台股2015年封關日最後一檔有神秘買盤進場，強拉加權指數上漲39點，元...   \n",
       "\n",
       "                   id p_type  \\\n",
       "30  1451618809219_N01   news   \n",
       "97  1451715108530_N01   news   \n",
       "\n",
       "                                             page_url           post_time  \\\n",
       "30  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-01 10:59:00   \n",
       "97  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-02 13:50:00   \n",
       "\n",
       "   s_area_name   s_name                     title  post_time2  \n",
       "30        重大要聞  yahoo股市       《熱門族群》12檔話題當紅股，完美封關  2016-01-01  \n",
       "97        重大要聞  yahoo股市  錢冠州：總統大選不再是大變數 元月行情伺機而動！  2016-01-02  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_company = df_article[ df_article['content'].str.contains('國巨')]   # df 某欄位 string contains \"國巨\"\n",
    "#df_company = df_article[ df_article['content'].str.contains('奇力新')] \n",
    "#df_company = df_article[ df_article['content'].str.contains('大毅')]  \n",
    "#df_company = df_article[ df_article['content'].str.contains('同欣電 ')]\n",
    "df_company = df_article[ df_article['content'].str.contains(company_words)]   \n",
    "print(len(df_company))\n",
    "del df_article\n",
    "df_company.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords=list()\n",
    "with open('stopword.goatwang.kang.txt', 'r',encoding='utf-8') as data:\n",
    "    for stopword in data:\n",
    "        stopwords.append(stopword.strip('\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home/barry/projects/stock/dict.idkrsi.txt ...\n",
      "Loading model from cache /tmp/jieba.ufc1c7fa02eb193cee2bb78b34d63c017.cache\n",
      "Loading model cost 0.395 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "# 'content'全部切詞\n",
    "corpus = []  # array\n",
    "for index, row in df_company.iterrows():  \n",
    "    not_cut = df_company.loc[index,'content']\n",
    "#     not_cut = row['description']                    # 跟上一行一樣意思\n",
    "    seg_generator = jieba.cut(not_cut, cut_all=False)  # genarator\n",
    "    seglist = list(seg_generator)                     # 整篇文章string切出來的list\n",
    "    seglist = list(filter(lambda a: a not in stopwords and a != '\\n', seglist ))  #去除停用詞 #未必需要這步驟\n",
    "    corpus.append(' '.join(seglist))          # ' '.join(seg_generator)也可\n",
    "\n",
    "df_company[\"content2\"]=corpus  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>content</th>\n",
       "      <th>id</th>\n",
       "      <th>p_type</th>\n",
       "      <th>page_url</th>\n",
       "      <th>post_time</th>\n",
       "      <th>s_area_name</th>\n",
       "      <th>s_name</th>\n",
       "      <th>title</th>\n",
       "      <th>post_time2</th>\n",
       "      <th>content2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>時報資訊</td>\n",
       "      <td>0</td>\n",
       "      <td>台股2015年下半年跌勢洶洶，全年跌幅達10.14％，但12檔具利多題材個股，股價多頭氣盛，...</td>\n",
       "      <td>1451618809219_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-01 10:59:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>《熱門族群》12檔話題當紅股，完美封關</td>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>台股 2015 下半年 跌勢 洶洶 跌幅 達 10.14 12 檔具 利多 題材 個股 股價...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>鉅亨網</td>\n",
       "      <td>0</td>\n",
       "      <td>鉅亨網記者王莞甯 台北台股2015年封關日最後一檔有神秘買盤進場，強拉加權指數上漲39點，元...</td>\n",
       "      <td>1451715108530_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-02 13:50:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>錢冠州：總統大選不再是大變數 元月行情伺機而動！</td>\n",
       "      <td>2016-01-02</td>\n",
       "      <td>鉅亨網 記者 王莞 甯 台北 台股 2015 封關日 最後 一檔 神秘 買盤 進場 強拉 加...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   author  comment_count                                            content  \\\n",
       "30   時報資訊              0  台股2015年下半年跌勢洶洶，全年跌幅達10.14％，但12檔具利多題材個股，股價多頭氣盛，...   \n",
       "97    鉅亨網              0  鉅亨網記者王莞甯 台北台股2015年封關日最後一檔有神秘買盤進場，強拉加權指數上漲39點，元...   \n",
       "\n",
       "                   id p_type  \\\n",
       "30  1451618809219_N01   news   \n",
       "97  1451715108530_N01   news   \n",
       "\n",
       "                                             page_url           post_time  \\\n",
       "30  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-01 10:59:00   \n",
       "97  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-02 13:50:00   \n",
       "\n",
       "   s_area_name   s_name                     title  post_time2  \\\n",
       "30        重大要聞  yahoo股市       《熱門族群》12檔話題當紅股，完美封關  2016-01-01   \n",
       "97        重大要聞  yahoo股市  錢冠州：總統大選不再是大變數 元月行情伺機而動！  2016-01-02   \n",
       "\n",
       "                                             content2  \n",
       "30  台股 2015 下半年 跌勢 洶洶 跌幅 達 10.14 12 檔具 利多 題材 個股 股價...  \n",
       "97  鉅亨網 記者 王莞 甯 台北 台股 2015 封關日 最後 一檔 神秘 買盤 進場 強拉 加...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_company.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trend.loc[2,'年月日'].date() +  timedelta(days=-1) == df_trend.loc[1,'年月日'].date()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trend.loc[5,'年月日'].date() +  timedelta(days=-1) == df_trend.loc[4,'年月日'].date()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.timedelta(1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = df_trend.loc[1,'年月日'].date() - df_trend.loc[ 1-1 ,'年月日'].date() #相減差幾天\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.days #只取天數"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "int(d.days) #幾天 轉整數"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.date(2016, 1, 7)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trend.loc[3,'年月日'].date() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>content</th>\n",
       "      <th>id</th>\n",
       "      <th>p_type</th>\n",
       "      <th>page_url</th>\n",
       "      <th>post_time</th>\n",
       "      <th>s_area_name</th>\n",
       "      <th>s_name</th>\n",
       "      <th>title</th>\n",
       "      <th>post_time2</th>\n",
       "      <th>content2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Moneydj理財網</td>\n",
       "      <td>0</td>\n",
       "      <td>1.週三台股在美股重挫以及陸股再度啟動融斷機制之下，指數重挫點，指數跌破十年線，所幸尾盤大型...</td>\n",
       "      <td>1452156767683_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-07 16:29:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>日盛投顧：台股短線有反彈機會</td>\n",
       "      <td>2016-01-07</td>\n",
       "      <td>週三 台股 美股 重挫 陸股 再度 啟動 融斷 機制 之下 指數 重挫 點 指數 跌破 十年...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         author  comment_count  \\\n",
       "999  Moneydj理財網              0   \n",
       "\n",
       "                                               content                 id  \\\n",
       "999  1.週三台股在美股重挫以及陸股再度啟動融斷機制之下，指數重挫點，指數跌破十年線，所幸尾盤大型...  1452156767683_N01   \n",
       "\n",
       "    p_type                                           page_url  \\\n",
       "999   news  https://tw.stock.yahoo.com/news_content/url/d/...   \n",
       "\n",
       "              post_time s_area_name   s_name           title  post_time2  \\\n",
       "999 2016-01-07 16:29:00        重大要聞  yahoo股市  日盛投顧：台股短線有反彈機會  2016-01-07   \n",
       "\n",
       "                                              content2  \n",
       "999  週三 台股 美股 重挫 陸股 再度 啟動 融斷 機制 之下 指數 重挫 點 指數 跌破 十年...  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_company[ df_company['post_time2'] == df_trend.loc[3,'年月日'].date() ].head() # 某欄位 == n 的 全部撈出來"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 演算法 \n",
    "# for index, row in df_2327.iterrows():\n",
    "#     try:\n",
    "#         if df_2327.loc[index,'年月日'].date() +  timedelta(days=-1) == df_2327.loc[index-1,'年月日'].date():                        \n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-1), 'label'] = df_2327.loc[index,'tag']\n",
    "#             # 如果股票前一筆差1天  # 那前1天的文章標上當天的漲跌   \n",
    "        \n",
    "#         elif df_2327.loc[index,'年月日'].date() +  timedelta(days=-2) == df_2327.loc[index-1,'年月日'].date():           \n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-1), 'label'] = df_2327.loc[index,'tag']\n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-2), 'label'] = df_2327.loc[index,'tag']\n",
    "#             # 如果股票前一筆差2天 #那前2天的文章標上當天的漲跌\n",
    "            \n",
    "#         elif df_2327.loc[index,'年月日'].date() +  timedelta(days=-3) == df_2327.loc[index-1,'年月日'].date():\n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-1), 'label'] = df_2327.loc[index,'tag']\n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-2), 'label'] = df_2327.loc[index,'tag']\n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-3), 'label'] = df_2327.loc[index,'tag']\n",
    "#         elif df_2327.loc[index,'年月日'].date() +  timedelta(days=-4) == df_2327.loc[index-1,'年月日'].date():\n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-1), 'label'] = df_2327.loc[index,'tag']\n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-2), 'label'] = df_2327.loc[index,'tag']\n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-3), 'label'] = df_2327.loc[index,'tag']\n",
    "#             df_forum.loc[df_forum['post_time2'] ==  df_2327.loc[index,'年月日'].date() +  timedelta(days=-4), 'label'] = df_2327.loc[index,'tag']\n",
    "\n",
    "\n",
    "#     except:\n",
    "#         continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 看所有相差的天數\n",
    "# for index, row in df_2327.iterrows():\n",
    "#     try:\n",
    "#         n = df_2327.loc[index,'年月日'].date() - df_2327.loc[index-1,'年月日'].date()\n",
    "#         print(n)\n",
    "#     except:\n",
    "#         continue\n",
    "\n",
    "# 最多12天"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如果股票前一筆差n天  # 那前n天的文章標上當天的漲跌 \n",
    "df_company['label5566']='--'\n",
    "for index, row in df_trend.iterrows():\n",
    "    try:\n",
    "        n = int((df_trend.loc[index,'年月日'].date() - df_trend.loc[index-1,'年月日'].date()).days ) # 差幾個datetime # 轉天數 # 再轉整數\n",
    "#        print(n)\n",
    "\n",
    "        for i in range(1, n+1):\n",
    "#            print(i)\n",
    "            df_company.loc[df_company['post_time2'] ==  df_trend.loc[index,'年月日'].date() +  timedelta(days=-i), 'label5566'] = df_trend.loc[index,'tag']\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1284\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>content</th>\n",
       "      <th>id</th>\n",
       "      <th>p_type</th>\n",
       "      <th>page_url</th>\n",
       "      <th>post_time</th>\n",
       "      <th>s_area_name</th>\n",
       "      <th>s_name</th>\n",
       "      <th>title</th>\n",
       "      <th>post_time2</th>\n",
       "      <th>content2</th>\n",
       "      <th>label5566</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1522</th>\n",
       "      <td>財訊快報</td>\n",
       "      <td>0</td>\n",
       "      <td>◆全台勞檢啟動，11類行業小心，其中284家電子零組件企業，實施全面普查。&lt;BR&gt; ◆海外分...</td>\n",
       "      <td>1452472319234_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-11 08:05:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>工商時報晨訊</td>\n",
       "      <td>2016-01-11</td>\n",
       "      <td>全台 勞檢 啟動 11 類 行業 小心 284 家 電子 零組件 企業 實施 全面 普查 海...</td>\n",
       "      <td>down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1530</th>\n",
       "      <td>時報資訊</td>\n",
       "      <td>0</td>\n",
       "      <td>陸股暫停熔斷機制，減緩市場對於系統性風險的疑慮，法人指出，台股短線壓力仍大，若結構出現惡化現...</td>\n",
       "      <td>1452472329341_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-11 08:15:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>《基金》全球經濟趨緩，慎選台股標的</td>\n",
       "      <td>2016-01-11</td>\n",
       "      <td>陸股 暫停 熔斷 機制 減緩 市場 系統性 風險 疑慮 法人 指出 台股 短線 壓力 仍大 ...</td>\n",
       "      <td>down</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     author  comment_count                                            content  \\\n",
       "1522   財訊快報              0  ◆全台勞檢啟動，11類行業小心，其中284家電子零組件企業，實施全面普查。<BR> ◆海外分...   \n",
       "1530   時報資訊              0  陸股暫停熔斷機制，減緩市場對於系統性風險的疑慮，法人指出，台股短線壓力仍大，若結構出現惡化現...   \n",
       "\n",
       "                     id p_type  \\\n",
       "1522  1452472319234_N01   news   \n",
       "1530  1452472329341_N01   news   \n",
       "\n",
       "                                               page_url           post_time  \\\n",
       "1522  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-11 08:05:00   \n",
       "1530  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-11 08:15:00   \n",
       "\n",
       "     s_area_name   s_name              title  post_time2  \\\n",
       "1522        重大要聞  yahoo股市             工商時報晨訊  2016-01-11   \n",
       "1530        重大要聞  yahoo股市  《基金》全球經濟趨緩，慎選台股標的  2016-01-11   \n",
       "\n",
       "                                               content2 label5566  \n",
       "1522  全台 勞檢 啟動 11 類 行業 小心 284 家 電子 零組件 企業 實施 全面 普查 海...      down  \n",
       "1530  陸股 暫停 熔斷 機制 減緩 市場 系統性 風險 疑慮 法人 指出 台股 短線 壓力 仍大 ...      down  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_company[df_company['label5566']=='down']))\n",
    "df_company[df_company['label5566']=='down'].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "891\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>content</th>\n",
       "      <th>id</th>\n",
       "      <th>p_type</th>\n",
       "      <th>page_url</th>\n",
       "      <th>post_time</th>\n",
       "      <th>s_area_name</th>\n",
       "      <th>s_name</th>\n",
       "      <th>title</th>\n",
       "      <th>post_time2</th>\n",
       "      <th>content2</th>\n",
       "      <th>label5566</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>財訊快報</td>\n",
       "      <td>0</td>\n",
       "      <td>◆千億科技預算，張善政喊加碼，主計總處初步匡列1 061億、年增3%，可能再調高。&lt;BR&gt; ...</td>\n",
       "      <td>1451867210012_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-04 07:56:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>工商時報晨訊</td>\n",
       "      <td>2016-01-04</td>\n",
       "      <td>千億 科技 預算 張善政 喊 加碼 主計 總處 初步 匡列 061 億 年增 3% 調高 產...</td>\n",
       "      <td>up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>時報資訊</td>\n",
       "      <td>0</td>\n",
       "      <td>2016年開紅盤日，台股狂洩逾200點！儘管2015年最後一個交易日拉尾盤，守住8300點，...</td>\n",
       "      <td>1451879808470_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-04 11:34:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>《盤中解析》紅盤日變臉，台股狂洩逾200點</td>\n",
       "      <td>2016-01-04</td>\n",
       "      <td>2016 年開 紅盤 台股 狂洩 逾 200 點 2015 最後 一個 交易日 拉 尾盤 守...</td>\n",
       "      <td>up</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    author  comment_count                                            content  \\\n",
       "222   財訊快報              0  ◆千億科技預算，張善政喊加碼，主計總處初步匡列1 061億、年增3%，可能再調高。<BR> ...   \n",
       "289   時報資訊              0  2016年開紅盤日，台股狂洩逾200點！儘管2015年最後一個交易日拉尾盤，守住8300點，...   \n",
       "\n",
       "                    id p_type  \\\n",
       "222  1451867210012_N01   news   \n",
       "289  1451879808470_N01   news   \n",
       "\n",
       "                                              page_url           post_time  \\\n",
       "222  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-04 07:56:00   \n",
       "289  https://tw.stock.yahoo.com/news_content/url/d/... 2016-01-04 11:34:00   \n",
       "\n",
       "    s_area_name   s_name                  title  post_time2  \\\n",
       "222        重大要聞  yahoo股市                 工商時報晨訊  2016-01-04   \n",
       "289        重大要聞  yahoo股市  《盤中解析》紅盤日變臉，台股狂洩逾200點  2016-01-04   \n",
       "\n",
       "                                              content2 label5566  \n",
       "222  千億 科技 預算 張善政 喊 加碼 主計 總處 初步 匡列 061 億 年增 3% 調高 產...        up  \n",
       "289  2016 年開 紅盤 台股 狂洩 逾 200 點 2015 最後 一個 交易日 拉 尾盤 守...        up  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_company[df_company['label5566']=='up']))\n",
    "df_company[df_company['label5566']=='up'].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "911\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>content</th>\n",
       "      <th>id</th>\n",
       "      <th>p_type</th>\n",
       "      <th>page_url</th>\n",
       "      <th>post_time</th>\n",
       "      <th>s_area_name</th>\n",
       "      <th>s_name</th>\n",
       "      <th>title</th>\n",
       "      <th>post_time2</th>\n",
       "      <th>content2</th>\n",
       "      <th>label5566</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3121</th>\n",
       "      <td>鉅亨網</td>\n",
       "      <td>0</td>\n",
       "      <td>受到月線反壓、外資籌碼面持續偏保守的影響下，台股期現貨今(20)日同步下壓，加權指數開低走低...</td>\n",
       "      <td>1453258680867_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-20 10:42:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>台股盤中─個股漲勢稀落 指數一度大跌百點 台指期難拉高結算</td>\n",
       "      <td>2016-01-20</td>\n",
       "      <td>月線 反壓 外資 籌碼面 持續 偏 保守 影響 台股 期 現貨 20 同步 下壓 加權 指數...</td>\n",
       "      <td>even</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3192</th>\n",
       "      <td>Moneydj理財網</td>\n",
       "      <td>0</td>\n",
       "      <td>1.歐美股市走穩，不過受期指結算影響，台股週三開低，加上港股重挫，指數震盪走低，盤面上部份生...</td>\n",
       "      <td>1453281180692_N01</td>\n",
       "      <td>news</td>\n",
       "      <td>https://tw.stock.yahoo.com/news_content/url/d/...</td>\n",
       "      <td>2016-01-20 16:16:00</td>\n",
       "      <td>重大要聞</td>\n",
       "      <td>yahoo股市</td>\n",
       "      <td>日盛投顧：台股短線參考區間7600-7900</td>\n",
       "      <td>2016-01-20</td>\n",
       "      <td>歐美 股市 走穩 受 期指 結算 影響 台股 週三 開低 港股 重挫 指數 震盪 走低 盤面...</td>\n",
       "      <td>even</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          author  comment_count  \\\n",
       "3121         鉅亨網              0   \n",
       "3192  Moneydj理財網              0   \n",
       "\n",
       "                                                content                 id  \\\n",
       "3121  受到月線反壓、外資籌碼面持續偏保守的影響下，台股期現貨今(20)日同步下壓，加權指數開低走低...  1453258680867_N01   \n",
       "3192  1.歐美股市走穩，不過受期指結算影響，台股週三開低，加上港股重挫，指數震盪走低，盤面上部份生...  1453281180692_N01   \n",
       "\n",
       "     p_type                                           page_url  \\\n",
       "3121   news  https://tw.stock.yahoo.com/news_content/url/d/...   \n",
       "3192   news  https://tw.stock.yahoo.com/news_content/url/d/...   \n",
       "\n",
       "               post_time s_area_name   s_name                          title  \\\n",
       "3121 2016-01-20 10:42:00        重大要聞  yahoo股市  台股盤中─個股漲勢稀落 指數一度大跌百點 台指期難拉高結算   \n",
       "3192 2016-01-20 16:16:00        重大要聞  yahoo股市         日盛投顧：台股短線參考區間7600-7900   \n",
       "\n",
       "      post_time2                                           content2 label5566  \n",
       "3121  2016-01-20  月線 反壓 外資 籌碼面 持續 偏 保守 影響 台股 期 現貨 20 同步 下壓 加權 指數...      even  \n",
       "3192  2016-01-20  歐美 股市 走穩 受 期指 結算 影響 台股 週三 開低 港股 重挫 指數 震盪 走低 盤面...      even  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(df_company[df_company['label5566']=='even']))\n",
    "df_company[df_company['label5566']=='even'].head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_company2 = df_company[df_company['label5566'].str.contains('up|down')]   #只取漲跌"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The maximum number of words to be used. (most frequent)\n",
    "MAX_NB_WORDS = 50000\n",
    "# Max number of words in each complaint.\n",
    "MAX_SEQUENCE_LENGTH = 1000\n",
    "# This is fixed.\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 下面的几条语句，会自动帮助我们，把分词之后的评论信息，转换成为一系列的数字组成的序列。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 34862 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~', lower=True)\n",
    "tokenizer.fit_on_texts(df_company2['content2'].to_numpy())\n",
    "\n",
    "word_index = tokenizer.word_index                         # 每個詞的索引\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "#print(word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X: (2175, 1000)\n"
     ]
    }
   ],
   "source": [
    "X = tokenizer.texts_to_sequences(df_company2['content2'].to_numpy())  # 轉成向量\n",
    "X = pad_sequences(X, maxlen=MAX_SEQUENCE_LENGTH)                      # 向量長短不一 統一化補0\n",
    "print('Shape of X:', X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import re\n",
    "# features = []   # features=list()\n",
    "# with open('finance.words.txt', 'r',encoding='utf-8') as data:\n",
    "#     for line in data:\n",
    "# #        line = re.sub('[a-zA-Z0-9\\W]', '', line)                     # 把數字英文去掉\n",
    "#         line = re.sub('[0-9]', '', line)                              # 把數字去掉\n",
    "#         features.append(line.replace('\\n', '').replace(' ', ''))      # 空格 \\n去掉\n",
    "\n",
    "# print(len(features))\n",
    "# print(type(features))\n",
    "# features[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Y: (2175, 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/barry/miniconda3/envs/stockenv/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "le = LabelEncoder()  # label 如果是string轉成整數\n",
    "df_company2['label5567'] = le.fit_transform(df_company2['label5566'])\n",
    "Y = pd.get_dummies(df_company2['label5567']).to_numpy() # label轉乘2維矩陣   # keras不吃1維label\n",
    "print('Shape of Y:', Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1740, 1000) (1740, 2)\n",
      "(435, 1000) (435, 2)\n"
     ]
    }
   ],
   "source": [
    "from sklearn import model_selection\n",
    "#將X:features array, Y:lable array 都切成 1:4\n",
    "validation_size = 0.20\n",
    "seed = 7\n",
    "X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=validation_size,\n",
    "                                                                                random_state=seed,\n",
    "                                                                                stratify=Y)\n",
    "print(X_train.shape, Y_train.shape)\n",
    "print(X_validation.shape, Y_validation.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 在Keras模型中使用预训练的词向量\n",
    "https://keras-cn-docs.readthedocs.io/zh_CN/latest/blog/word_embedding/\n",
    "\n",
    "### Embedding layer设置\n",
    "### 下載的詞向量文件中解析出每个词和它所对应的词向量，并用字典的方式存储\n",
    "\n",
    "#### 詞向量載點\n",
    "https://fasttext.cc/docs/en/crawl-vectors.html\n",
    "\n",
    "https://github.com/lintseju/word_embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2000000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "embeddings_index = {}\n",
    "f = open('cc.zh.300.vec', 'r', encoding='utf-8')\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 此时，我们可以根据得到的字典生成上文所定义的词向量矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(34863, 300)\n",
      "[[ 0.00000000e+00  0.00000000e+00  0.00000000e+00 ...  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00]\n",
      " [ 6.26000017e-02  2.85000000e-02  4.48900014e-01 ...  8.29999987e-03\n",
      "   8.51000026e-02  7.64999986e-02]\n",
      " [ 5.53999990e-02  2.26000007e-02  5.66999972e-01 ...  1.97799996e-01\n",
      "  -5.62999994e-02  3.26999985e-02]\n",
      " ...\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00 ...  0.00000000e+00\n",
      "   0.00000000e+00  0.00000000e+00]\n",
      " [-3.99999990e-04 -1.57900006e-01  2.54599988e-01 ... -7.50000030e-02\n",
      "   3.44000012e-02 -8.16999972e-02]\n",
      " [ 1.90000003e-03  1.06100000e-01  1.66999996e-01 ... -1.09999999e-01\n",
      "  -1.62599996e-01  7.84000009e-02]]\n"
     ]
    }
   ],
   "source": [
    "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "print(embedding_matrix.shape)\n",
    "print(embedding_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 另一個讀取詞向量的方法\n",
    "https://sspai.com/post/49121\n",
    "\n",
    "https://github.com/wshuyi/demo-chinese-text-classification-lstm-keras/blob/master/demo.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from gensim.models import KeyedVectors\n",
    "\n",
    "# zh_model = KeyedVectors.load_word2vec_format('zh_wiki_fasttext_300.txt')\n",
    "\n",
    "# zh_model.vectors[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(iter(zh_model.vocab))[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(zh_model[next(iter(zh_model.vocab))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding_dim = len(zh_model[next(iter(zh_model.vocab))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding_matrix = np.random.rand(MAX_NB_WORDS, EMBEDDING_DIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding_matrix = (embedding_matrix - 0.5) * 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# zh_model.get_vector('的')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for word, i in word_index.items():\n",
    "#     if i < MAX_NB_WORDS:\n",
    "#         try:\n",
    "#           embedding_vector = zh_model.get_vector(word)\n",
    "#           embedding_matrix[i] = embedding_vector\n",
    "#         except:\n",
    "#           pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 现在我们将这个词向量矩阵加载到Embedding层中，注意，我们设置trainable=False使得这个编码层不可再训练。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/barry/miniconda3/envs/stockenv/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/barry/miniconda3/envs/stockenv/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 1000, 300)         10458900  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 256)               439296    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 10,947,862\n",
      "Trainable params: 488,962\n",
      "Non-trainable params: 10,458,900\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "## LSTM\n",
    "#inputs = Input(name='inputs',shape=[MAX_SEQUENCE_LENGTH])\n",
    "model = Sequential()\n",
    "## Embedding(词汇表大小,batch大小,每个新闻的词长)\n",
    "# model.add( Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH) )\n",
    "\n",
    "model.add( Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False))\n",
    "model.add( Bidirectional(LSTM(128, return_sequences=False )) )\n",
    "model.add( Dense(128,activation=\"relu\") )\n",
    "model.add( Dense(128,activation=\"relu\") )\n",
    "model.add( Dropout(0.5))\n",
    "model.add( Dense(2,activation=\"softmax\") )\n",
    "#model = Model(inputs=inputs,outputs=layer)\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss=\"categorical_crossentropy\",\n",
    "              optimizer='adam', # optimizer=Adam(lr=0.0001, decay=1e-6),\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/barry/miniconda3/envs/stockenv/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Train on 1740 samples, validate on 435 samples\n",
      "Epoch 1/20\n",
      "1740/1740 [==============================] - 64s 37ms/step - loss: 0.6806 - acc: 0.5891 - val_loss: 0.6787 - val_acc: 0.5908\n",
      "Epoch 2/20\n",
      "1740/1740 [==============================] - 61s 35ms/step - loss: 0.6710 - acc: 0.5914 - val_loss: 0.6784 - val_acc: 0.5885\n",
      "Epoch 3/20\n",
      "1740/1740 [==============================] - 61s 35ms/step - loss: 0.6605 - acc: 0.6121 - val_loss: 0.6779 - val_acc: 0.5678\n",
      "Epoch 4/20\n",
      "1740/1740 [==============================] - 61s 35ms/step - loss: 0.6341 - acc: 0.6402 - val_loss: 0.7015 - val_acc: 0.5678\n"
     ]
    }
   ],
   "source": [
    "model_fit = model.fit(X_train, Y_train, batch_size=64,epochs=20,\n",
    "                      validation_data=(X_validation,Y_validation),\n",
    "                      callbacks=[EarlyStopping(monitor='val_loss',min_delta=0.0001)] ## 当val-loss不再提升时停止训练\n",
    "                     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, ..., 1, 1, 1])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(Y_train,axis=1) # 轉成1維"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5678160919540229\n",
      "[[183  74]\n",
      " [114  64]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.62      0.71      0.66       257\n",
      "           1       0.46      0.36      0.41       178\n",
      "\n",
      "   micro avg       0.57      0.57      0.57       435\n",
      "   macro avg       0.54      0.54      0.53       435\n",
      "weighted avg       0.55      0.57      0.56       435\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict(X_validation)\n",
    "print(accuracy_score(np.argmax(Y_validation,axis=1), np.argmax(predictions,axis=1)))\n",
    "print(confusion_matrix(np.argmax(Y_validation,axis=1), np.argmax(predictions,axis=1)))\n",
    "print(classification_report(np.argmax(Y_validation,axis=1), np.argmax(predictions,axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}