Lesson10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据载入\n",
    "- [dataset](https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('breast-cancer-wisconsin.data', names=[\n",
    "    'Sample code number',\n",
    "    'Clump Thickness',\n",
    "    'Uniformity of Cell Size',\n",
    "    'Uniformity of Cell Shape',\n",
    "    'Marginal Adhesion',\n",
    "    'Single Epithelial Cell Size'\n",
    "    'Bare Nuclei',\n",
    "    'Bland Chromatin',\n",
    "    'Normal Nucleoli',\n",
    "    'Mitoses',\n",
    "    'Class'\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = df.replace(to_replace='?', value=np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.dropna(how='any', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(df[df.columns[0:9]], df[df.columns[9]], test_size=0.25, random_state=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_test.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import SGDClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train = ss.fit_transform(X_train)\n",
    "X_test = ss.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression()\n",
    "sgdb = SGDClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_y_predict = lr.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_y_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sgdb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sgdb_y_predict = sgdb.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sgdb_y_predict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 性能评测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print 'Accuracy of LR', lr.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print classification_report(y_test, lr_y_predict, target_names=['Benign', 'Malignant'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print 'Accuracy of SGDC', sgdb.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print classification_report(y_test, sgdb_y_predict, target_names=['Benign', 'Malignant'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 手写体 SVM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "digits = load_digits()\n",
    "digits.data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, test_size=0.25, random_state=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train = ss.fit_transform(X_train)\n",
    "X_test = ss.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lsvc = LinearSVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lsvc.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_predict = lsvc.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print 'Accuracy of SVM', lsvc.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print classification_report(y_test, y_predict, target_names=digits.target_names.astype(str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 监督方法\n",
    "## KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iris = load_iris()\n",
    "iris.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iris.DESCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.25, random_state=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train = ss.fit_transform(X_train)\n",
    "X_test = ss.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "knc = KNeighborsClassifier()\n",
    "knc.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_predict = knc.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print 'Accuracy of K-Nearest Neighbor', knc.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print classification_report(y_test, y_predict, target_names=digits.target_names.astype(str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K-Means\n",
    "- [dataset](https://archive.ics.uci.edu/ml/machine-learning-databases/optdigits/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "digits_train = pd.read_csv('optdigits.tra', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "digits_test = pd.read_csv('optdigits.tes', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = digits_train[np.arange(64)]\n",
    "y_train = digits_train[64]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test = digits_test[np.arange(64)]\n",
    "y_test = digits_test[64]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kmeans = KMeans(n_clusters=10)\n",
    "kmeans.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = kmeans.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print metrics.adjusted_rand_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sent1 = 'The cat is walking in the bedroom'\n",
    "sent2 = 'A dog was running across the kitchen.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "count_vec = CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentences = [sent1, sent2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print count_vec.fit_transform(sentences).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print count_vec.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "#nltk.download() # download tokenizers/punkt/english.pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tokens_l = nltk.word_tokenize(sent1)\n",
    "print tokens_l"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Titanic\n",
    "\n",
    "- [dataset](https://www.kaggle.com/c/digit-recognizer/data)\n",
    "\n",
    "Variable\tDefinition\tKey\n",
    "survival\tSurvival\t0 = No, 1 = Yes\n",
    "pclass\tTicket class\t1 = 1st, 2 = 2nd, 3 = 3rd\n",
    "sex\tSex\t\n",
    "Age\tAge in years\t\n",
    "sibsp\t# of siblings / spouses aboard the Titanic\t\n",
    "parch\t# of parents / children aboard the Titanic\t\n",
    "ticket\tTicket number\t\n",
    "fare\tPassenger fare\t\n",
    "cabin\tCabin number\t\n",
    "embarked\tPort of Embarkation\tC = Cherbourg, Q = Queenstown, S = Southampton\n",
    "\n",
    "Variable Notes\n",
    "\n",
    "pclass: A proxy for socio-economic status (SES)\n",
    "1st = Upper\n",
    "2nd = Middle\n",
    "3rd = Lower\n",
    "\n",
    "age: Age is fractional if less than 1. If the age is estimated, is it in the form of xx.5\n",
    "\n",
    "sibsp: The dataset defines family relations in this way...\n",
    "Sibling = brother, sister, stepbrother, stepsister\n",
    "Spouse = husband, wife (mistresses and fiancés were ignored)\n",
    "\n",
    "parch: The dataset defines family relations in this way...\n",
    "Parent = mother, father\n",
    "Child = daughter, son, stepdaughter, stepson\n",
    "Some children travelled only with a nanny, therefore parch=0 for them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('titanic/train.csv')\n",
    "#print train.info()\n",
    "test = pd.read_csv('titanic/test.csv')\n",
    "#print test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = ['Pclass','Sex','Age','SibSp','Parch','Embarked','Fare']\n",
    "X_train = train[features]\n",
    "X_test = test[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train['Age'].fillna(train['Age'].mean(), inplace=True)\n",
    "X_train['Embarked'].fillna('S', inplace=True)\n",
    "X_test['Age'].fillna(train['Age'].mean(), inplace=True)\n",
    "X_test['Embarked'].fillna('S', inplace=True)\n",
    "\n",
    "y_train = train['Survived']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction import DictVectorizer\n",
    "dict_vec = DictVectorizer(sparse=False)\n",
    "X_train = dict_vec.fit_transform(X_train.to_dict(orient='record'))\n",
    "print dict_vec.feature_names_\n",
    "\n",
    "print X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test = dict_vec.transform(X_test.to_dict(orient='record'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rfc = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#cross_val_score(rfc, X_train, y_train, cv=5).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cross_val_score(rfc, X_train, y_train, cv=5).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#print test['PassengerId']\n",
    "rfc.fit(X_train, y_train)\n",
    "#rfc_submission = pd.DataFrame({'PassengerId':test['PassengerId'], 'Survived':rfc_y_predict})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test = X_test.astype(np.int)\n",
    "# rfc_y_predict = rfc.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rfc_y_predict = rfc.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rfc_y_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rfc_submisson = pd.DataFrame({'PassengerId': test['PassengerId'], 'Survived': rfc_y_predict})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rfc_submisson.to_csv('rfc_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMDB\n",
    "- [dataset](https://www.kaggle.com/c/word2vec-nlp-tutorial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from bs4 import BeautifulSoup "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('imdb/labeledTrainData.tsv', header=0,\n",
    "                delimiter=\"\\t\", quoting=3)\n",
    "test = pd.read_csv('imdb/testData.tsv', header=0, delimiter=\"\\t\",\n",
    "               quoting=3 )\n",
    "               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train['sentiment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def review_to_wordlist(review):\n",
    "    '''\n",
    "    Meant for converting each of the IMDB reviews into a list of words.\n",
    "    '''\n",
    "    # First remove the HTML.\n",
    "    review_text = BeautifulSoup(review, \"html5lib\").get_text()\n",
    "    \n",
    "    # Use regular expressions to only include words.\n",
    "    review_text = re.sub(\"[^a-zA-Z]\",\" \", review_text)\n",
    "    \n",
    "    # Convert words to lower case and split them into separate words.\n",
    "    words = review_text.lower().split()\n",
    "   \n",
    "    # Return a list of words\n",
    "    return(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print train.head()\n",
    "traindata = []\n",
    "for i in xrange(0,len(train['review'])):\n",
    "    traindata.append(\" \".join(review_to_wordlist(train['review'][i])))\n",
    "testdata = []\n",
    "print test.head()\n",
    "for i in xrange(0,len(test['review'])):\n",
    "    testdata.append(\" \".join(review_to_wordlist(test['review'][i])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer as TFIV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfv = TFIV(min_df=3,  max_features=None, \n",
    "        strip_accents='unicode', analyzer='word',token_pattern=r'\\w{1,}',\n",
    "        ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1,\n",
    "        stop_words = 'english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_all = traindata + testdata # Combine both to fit the TFIDF vectorization.\n",
    "lentrain = len(traindata)\n",
    "\n",
    "tfv.fit(X_all) # This is the slow part!\n",
    "X_all = tfv.transform(X_all)\n",
    "\n",
    "X = X_all[:lentrain] # Separate back into training and test sets. \n",
    "X_test = X_all[lentrain:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression as LR\n",
    "from sklearn.grid_search import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grid_values = {'C':[30]} # Decide which settings you want for the grid search. \n",
    "\n",
    "model_LR = GridSearchCV(LR(dual = True, random_state = 0), \n",
    "                        grid_values, scoring = 'roc_auc', cv = 20) \n",
    "# Try to set the scoring on what the contest is asking for. \n",
    "# The contest says scoring is for area under the ROC curve, so use this.\n",
    "                        \n",
    "model_LR.fit(X,y_train) # Fit the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_LR.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_LR.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('minst/train.csv')\n",
    "test = pd.read_csv('minst/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(42000, 785)\n",
      "(28000, 784)\n"
     ]
    }
   ],
   "source": [
    "print train.shape\n",
    "print test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = train['label']\n",
    "X_train = train.drop('label', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test = test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load or Download MNIST > data/mnist/\n",
      "data/mnist/train-images-idx3-ubyte.gz\n",
      "data/mnist/t10k-images-idx3-ubyte.gz\n",
      "  [TL] InputLayer  input_layer: (?, 784)\n",
      "  [TL] DropoutLayer drop1: keep:0.800000 is_fix:False\n",
      "  [TL] DenseLayer  relu1: 800 relu\n",
      "  [TL] DropoutLayer drop2: keep:0.500000 is_fix:False\n",
      "  [TL] DenseLayer  relu2: 800 relu\n",
      "  [TL] DropoutLayer drop3: keep:0.500000 is_fix:False\n",
      "  [TL] DenseLayer  output_layer: 10 identity\n",
      "  param   0: relu1/W:0            (784, 800)         float32_ref (mean: -0.000156755762873, median: -0.000153482658789, std: 0.0879837945104   )   \n",
      "  param   1: relu1/b:0            (800,)             float32_ref (mean: 0.0               , median: 0.0               , std: 0.0               )   \n",
      "  param   2: relu2/W:0            (800, 800)         float32_ref (mean: -5.23396010976e-05, median: -9.0194385848e-05 , std: 0.0879286974669   )   \n",
      "  param   3: relu2/b:0            (800,)             float32_ref (mean: 0.0               , median: 0.0               , std: 0.0               )   \n",
      "  param   4: output_layer/W:0     (800, 10)          float32_ref (mean: -0.000112081870611, median: -0.000939153658692, std: 0.0878978148103   )   \n",
      "  param   5: output_layer/b:0     (10,)              float32_ref (mean: 0.0               , median: 0.0               , std: 0.0               )   \n",
      "  num of params: 1276810\n",
      "  layer   0: drop1/mul:0          (?, 784)           float32\n",
      "  layer   1: relu1/Relu:0         (?, 800)           float32\n",
      "  layer   2: drop2/mul:0          (?, 800)           float32\n",
      "  layer   3: relu2/Relu:0         (?, 800)           float32\n",
      "  layer   4: drop3/mul:0          (?, 800)           float32\n",
      "  layer   5: output_layer/Identity:0 (?, 10)            float32\n",
      "Start training the network ...\n",
      "Epoch 1 of 500 took 4.844463s\n",
      "   val loss: 0.580788\n",
      "   val acc: 0.815400\n",
      "Epoch 5 of 500 took 4.839795s\n",
      "   val loss: 0.293327\n",
      "   val acc: 0.914300\n",
      "Epoch 10 of 500 took 4.830770s\n",
      "   val loss: 0.223286\n",
      "   val acc: 0.937200\n",
      "Epoch 15 of 500 took 4.855958s\n",
      "   val loss: 0.187789\n",
      "   val acc: 0.949300\n",
      "Epoch 20 of 500 took 4.826501s\n",
      "   val loss: 0.164285\n",
      "   val acc: 0.954800\n",
      "Epoch 25 of 500 took 4.841586s\n",
      "   val loss: 0.145372\n",
      "   val acc: 0.959400\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorlayer as tl\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "# 准备数据\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = \\\n",
    "                                tl.files.load_mnist_dataset(shape=(-1,784))\n",
    "\n",
    "# 定义 placeholder\n",
    "x = tf.placeholder(tf.float32, shape=[None, 784], name='x')\n",
    "y_ = tf.placeholder(tf.int64, shape=[None, ], name='y_')\n",
    "\n",
    "# 定义模型\n",
    "network = tl.layers.InputLayer(x, name='input_layer')\n",
    "network = tl.layers.DropoutLayer(network, keep=0.8, name='drop1')\n",
    "network = tl.layers.DenseLayer(network, n_units=800,\n",
    "                                act = tf.nn.relu, name='relu1')\n",
    "network = tl.layers.DropoutLayer(network, keep=0.5, name='drop2')\n",
    "network = tl.layers.DenseLayer(network, n_units=800,\n",
    "                                act = tf.nn.relu, name='relu2')\n",
    "network = tl.layers.DropoutLayer(network, keep=0.5, name='drop3')\n",
    "network = tl.layers.DenseLayer(network, n_units=10,\n",
    "                                act = tf.identity,\n",
    "                                name='output_layer')\n",
    "# 定义损失函数和衡量指标\n",
    "# tl.cost.cross_entropy 在内部使用 tf.nn.sparse_softmax_cross_entropy_with_logits() 实现 softmax\n",
    "y = network.outputs\n",
    "cost = tl.cost.cross_entropy(y, y_, name = 'cost')\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), y_)\n",
    "acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "y_op = tf.argmax(tf.nn.softmax(y), 1)\n",
    "\n",
    "# 定义 optimizer\n",
    "train_params = network.all_params\n",
    "train_op = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.9, beta2=0.999,\n",
    "                            epsilon=1e-08, use_locking=False).minimize(cost, var_list=train_params)\n",
    "\n",
    "# 初始化 session 中的所有参数\n",
    "tl.layers.initialize_global_variables(sess)\n",
    "\n",
    "# 列出模型信息\n",
    "network.print_params()\n",
    "network.print_layers()\n",
    "\n",
    "# 训练模型\n",
    "tl.utils.fit(sess, network, train_op, cost, X_train, y_train, x, y_,\n",
    "            acc=acc, batch_size=500, n_epoch=500, print_freq=5,\n",
    "            X_val=X_val, y_val=y_val, eval_train=False)\n",
    "\n",
    "# 评估模型\n",
    "tl.utils.test(sess, network, acc, X_test, y_test, x, y_, batch_size=None, cost=cost)\n",
    "\n",
    "# 把模型保存成 .npz 文件\n",
    "tl.files.save_npz(network.all_params , name='model.npz')\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}