CNN/Toy_model/5. mini_GoogLeNet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.sys.path.append(os.path.dirname(os.path.abspath('..')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype uint8 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 3072) (50000,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/utils/validation.py:595: DataConversionWarning: Data with input dtype uint8 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 3072) (10000,)\n"
     ]
    }
   ],
   "source": [
    "from dataset.dataset import load_cifar10\n",
    "\n",
    "train_data, test_data = load_cifar10(batch_size=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 网络结构设计\n",
    "GoogLeNet结构设计主要在于其独特的Interception结构：并行的三个卷积与最大池化操作。同样由于这里使用CIFAR10数据集的原因，这里实现的GoogLeNet是一个缩水版。首先将Interception之前的所有层替换成单层的conv+maxpooling。并且这里只设置4个Interception结构。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "unit_I = train_data.n_features    # 输入单元数，等于特征数\n",
    "\n",
    "# 三种卷积核\n",
    "conv_size1 = (1, 1)\n",
    "conv_size3 = (3, 3)\n",
    "conv_size5 = (5, 5)\n",
    "\n",
    "filters1 = 32\n",
    "filters_2 = [16, 64, 8]    # inception模块的通道列表\n",
    "filters_3 = [32, 128, 16]\n",
    "\n",
    "# 两种步长\n",
    "strides_1 = (1, 1)\n",
    "strides_2 = (2, 2)\n",
    "\n",
    "fc_size = 128    # 全连接层单元数\n",
    "\n",
    "unit_O = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义创建Interception结构的函数："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inc_block(inputs, channels, name=None):\n",
    "    '''\n",
    "    channels: 列表，包含三个卷积操作的输出深度\n",
    "    '''\n",
    "    in_channels = inputs.get_shape().as_list()[-1]    # 最后一维为深度\n",
    "\n",
    "    with tf.variable_scope(name):\n",
    "        conv1 = tf.layers.conv2d(inputs, filters=channels[0],\n",
    "                                 kernel_size=conv_size1, strides=strides_1,\n",
    "                                 padding='same', activation=tf.nn.relu)\n",
    "\n",
    "        conv2_1 = tf.layers.conv2d(inputs, filters=in_channels//2,    # 瓶颈层将参数减半\n",
    "                                   kernel_size=conv_size1, strides=strides_1,\n",
    "                                   padding='same', activation=tf.nn.relu)\n",
    "        conv2_2 = tf.layers.conv2d(conv2_1, filters=channels[1],\n",
    "                                   kernel_size=conv_size3, strides=strides_1,\n",
    "                                   padding='same', activation=tf.nn.relu)\n",
    "\n",
    "        conv3_1 = tf.layers.conv2d(inputs, filters=in_channels//2,    # 瓶颈层将参数减半\n",
    "                                   kernel_size=conv_size1, strides=strides_1,\n",
    "                                   padding='same', activation=tf.nn.relu)\n",
    "        conv3_2 = tf.layers.conv2d(conv3_1, filters=channels[2],\n",
    "                                   kernel_size=conv_size5, strides=strides_1,\n",
    "                                   padding='same', activation=tf.nn.relu)\n",
    "\n",
    "        pool4_1 = tf.layers.max_pooling2d(inputs, pool_size=conv_size3,\n",
    "                                          strides=strides_1, padding='same')\n",
    "        conv4_2 = tf.layers.conv2d(pool4_1, filters=in_channels//2,\n",
    "                                   kernel_size=conv_size1, strides=strides_1,\n",
    "                                   padding='same', activation=tf.nn.relu)\n",
    "\n",
    "    return tf.concat([conv1, conv2_2, conv3_2, conv4_2], axis=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 搭建网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-dc13b5c424b1>:10: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.conv2d instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From <ipython-input-5-dc13b5c424b1>:12: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.max_pooling2d instead.\n",
      "WARNING:tensorflow:From <ipython-input-5-dc13b5c424b1>:24: average_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.average_pooling2d instead.\n",
      "WARNING:tensorflow:From <ipython-input-5-dc13b5c424b1>:26: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.flatten instead.\n",
      "WARNING:tensorflow:From <ipython-input-5-dc13b5c424b1>:27: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/losses/losses_impl.py:209: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "X = tf.placeholder(tf.float32, [None, unit_I])  # 数据的样本数不指定，只指定特征数\n",
    "Y = tf.placeholder(tf.int64, [None])    # 目标值为列向量，int64为了兼容\n",
    "X_img = tf.transpose(tf.reshape(X, [-1, 3, 32, 32]),\n",
    "                     perm=[0, 2, 3, 1])    # 转为图片格式送入模型，(n_samples,width,height,depth)\n",
    "\n",
    "# 网络结构图\n",
    "with tf.name_scope('GoogLeNet'):\n",
    "    conv1 = tf.layers.conv2d(X_img, filters=32,\n",
    "                             kernel_size=conv_size5, padding='same',\n",
    "                             activation=tf.nn.relu, name='conv1')\n",
    "    pooling1 = tf.layers.max_pooling2d(conv1, pool_size=conv_size3,\n",
    "                                       strides=strides_2, name='pooling1')\n",
    "\n",
    "    inc_mod_2a = inc_block(pooling1, filters_2, 'inception_2a')\n",
    "    inc_mod_2b = inc_block(inc_mod_2a, filters_2, 'inception_2b')\n",
    "\n",
    "    pooling2 = tf.layers.max_pooling2d(inc_mod_2b, pool_size=conv_size3,\n",
    "                                       strides=strides_2, name='pooling2')\n",
    "\n",
    "    inc_mod_3a = inc_block(pooling2, filters_3, 'inception_3a')\n",
    "    inc_mod_3b = inc_block(inc_mod_3a, filters_3, 'inception_3b')\n",
    "\n",
    "    pooling3 = tf.layers.average_pooling2d(inc_mod_3b, pool_size=conv_size3,\n",
    "                                           strides=strides_1, name='pooling3')\n",
    "\n",
    "    fc = tf.layers.dense(tf.layers.flatten(pooling3), fc_size,\n",
    "                         activation=tf.nn.relu, name='FC')\n",
    "\n",
    "    logits = tf.layers.dense(fc, unit_O, activation=None)\n",
    "\n",
    "# 评估图\n",
    "with tf.name_scope('Eval'):\n",
    "    # 计算一维向量与onehot向量之间的损失\n",
    "    loss = tf.losses.sparse_softmax_cross_entropy(labels=Y, logits=logits)\n",
    "    predict = tf.argmax(logits, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(tf.equal(predict, Y), tf.float32))\n",
    "\n",
    "# 优化图\n",
    "with tf.name_scope('train_op'):\n",
    "    lr = 1e-3\n",
    "    train_op = tf.train.AdamOptimizer(lr).minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True    # 按需使用显存"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, batch_loss: 1.1947743892669678, batch_acc: 0.578125\n",
      "epoch: 2, batch_loss: 0.9500958919525146, batch_acc: 0.6875\n",
      "epoch: 3, batch_loss: 0.7968510389328003, batch_acc: 0.703125\n",
      "epoch: 5, batch_loss: 0.5155200362205505, batch_acc: 0.875\n",
      "epoch: 6, batch_loss: 0.7560532093048096, batch_acc: 0.71875\n",
      "epoch: 6, test_acc: 0.7427884340286255\n",
      "epoch: 7, batch_loss: 0.4780675172805786, batch_acc: 0.8125\n",
      "epoch: 8, batch_loss: 0.5322756767272949, batch_acc: 0.765625\n",
      "epoch: 10, batch_loss: 0.29794013500213623, batch_acc: 0.921875\n",
      "epoch: 11, batch_loss: 0.3662758469581604, batch_acc: 0.875\n",
      "epoch: 12, batch_loss: 0.34177935123443604, batch_acc: 0.890625\n",
      "epoch: 12, test_acc: 0.7739382982254028\n",
      "epoch: 14, batch_loss: 0.1486511379480362, batch_acc: 0.953125\n",
      "epoch: 15, batch_loss: 0.14802637696266174, batch_acc: 0.953125\n",
      "epoch: 16, batch_loss: 0.2157987654209137, batch_acc: 0.953125\n",
      "epoch: 17, batch_loss: 0.44383519887924194, batch_acc: 0.875\n",
      "epoch: 19, batch_loss: 0.06459574401378632, batch_acc: 0.96875\n",
      "epoch: 19, test_acc: 0.7702323794364929\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "with tf.Session(config=config) as sess:\n",
    "    sess.run(init)\n",
    "    epochs = 20\n",
    "\n",
    "    batch_cnt = 0\n",
    "    for epoch in range(epochs):\n",
    "        for batch_data, batch_labels in train_data.next_batch():\n",
    "            batch_cnt += 1\n",
    "            loss_val, acc_val, _ = sess.run([loss, accuracy, train_op],\n",
    "                                            feed_dict={X: batch_data,\n",
    "                                                       Y: batch_labels})\n",
    "\n",
    "            # 每1000batch输出一次信息\n",
    "            if (batch_cnt+1) % 1000 == 0:\n",
    "                print('epoch: {}, batch_loss: {}, batch_acc: {}'.format(\n",
    "                    epoch, loss_val, acc_val))\n",
    "\n",
    "            # 每5000batch做一次验证\n",
    "            if (batch_cnt+1) % 5000 == 0:\n",
    "                all_test_acc_val = list()\n",
    "                for test_batch_data, test_batch_labels in test_data.next_batch():\n",
    "                    test_acc_val = sess.run([accuracy],\n",
    "                                            feed_dict={X: test_batch_data,\n",
    "                                                       Y: test_batch_labels})\n",
    "                    all_test_acc_val.append(test_acc_val)\n",
    "                test_acc = np.mean(all_test_acc_val)\n",
    "                print('epoch: {}, test_acc: {}'.format(epoch, test_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里我们只构建了一个具有四个inception model的浅层GoogLeNet，可以看到其效果已经与之前的ResNet34相当。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}