m_ch03.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 2)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = np.array([[1,2], [3,4]])\n",
    "A.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 2)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B = np.array([[5,6], [7,8]])\n",
    "B.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[19, 22],\n",
       "       [43, 50]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(A,B)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3.3 ニューラルネットワークの行列の積"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = np.array([1, 2])\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 3 5]\n",
      " [2 4 6]]\n"
     ]
    }
   ],
   "source": [
    "W = np.array([[1,3,5], [2,4,6]])\n",
    "print(W)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 5 11 17]\n"
     ]
    }
   ],
   "source": [
    "Y = np.dot(X, W)\n",
    "print(Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 各層における信号伝達の実装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 3)\n",
      "(2,)\n",
      "(3,)\n"
     ]
    }
   ],
   "source": [
    "X = np.array([1.0, 0.5])\n",
    "W1 = np.array([[0.1, 0.3, 0.5], [0.2, 0.4, 0.6]])\n",
    "B1 = np.array([0.1, 0.2, 0.3])\n",
    "\n",
    "print(W1.shape)\n",
    "print(X.shape)\n",
    "print(B1.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.3 0.7 1.1]\n",
      "[0.57444252 0.66818777 0.75026011]\n"
     ]
    }
   ],
   "source": [
    "A1 = np.dot(X, W1) + B1\n",
    "Z1 = sigmoid(A1)\n",
    "\n",
    "print(A1)\n",
    "print(Z1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3,)\n",
      "(3, 2)\n",
      "(2,)\n"
     ]
    }
   ],
   "source": [
    "W2 = np.array([[0.1, 0.4], [0.2, 0.5], [0.3, 0.6]])\n",
    "B2 = np.array([0.1, 0.2])\n",
    "\n",
    "print(Z1.shape)\n",
    "print(W2.shape)\n",
    "print(B2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.51615984 1.21402696]\n",
      "[0.62624937 0.7710107 ]\n"
     ]
    }
   ],
   "source": [
    "A2 = np.dot(Z1, W2) + B2\n",
    "Z2 = sigmoid(A2)\n",
    "\n",
    "print(A2)\n",
    "print(Z2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.31682708, 0.69627909])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def identity_function(x):\n",
    "    return x\n",
    "\n",
    "W3 = np.array([[0.1, 0.3], [0.2, 0.4]])\n",
    "B3 = np.array([0.1, 0.2])\n",
    "\n",
    "A3 = np.dot(Z2, W3)+ B3\n",
    "Y = identity_function(A3)\n",
    "Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.31682708 0.69627909]\n"
     ]
    }
   ],
   "source": [
    "def init_network():\n",
    "    network = {}\n",
    "    network['W1'] = np.array([[0.1, 0.3, 0.5], [0.2, 0.4, 0.6]])\n",
    "    network['b1'] = np.array([0.1, 0.2, 0.3])\n",
    "    network['W2'] = np.array([[0.1, 0.4], [0.2, 0.5], [0.3, 0.6]])\n",
    "    network['b2'] = np.array([0.1, 0.2])\n",
    "    network['W3'] = np.array([[0.1, 0.3], [0.2, 0.4]])\n",
    "    network['b3'] = np.array([0.1, 0.2])\n",
    "    \n",
    "    return network\n",
    "\n",
    "def forward(network, x):\n",
    "    W1, W2, W3 = network['W1'],  network['W2'],  network['W3']\n",
    "    b1, b2, b3 = network['b1'],  network['b2'],  network['b3']\n",
    "    \n",
    "    a1 = np.dot(x, W1) + b1\n",
    "    z1 = sigmoid(a1)\n",
    "    a2 = np.dot(z1, W2) + b2\n",
    "    z2 = sigmoid(a2)\n",
    "    a3 = np.dot(z2, W3) + b3\n",
    "    y = identity_function(a3)\n",
    "    \n",
    "    return y\n",
    "\n",
    "network = init_network()\n",
    "x = np.array([1.0, 0.5])\n",
    "y = forward(network, x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.1 恒等関数とソフトマックス関数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([0.3, 2.9, 4.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.34985881 18.17414537 54.59815003]\n"
     ]
    }
   ],
   "source": [
    "exp_a = np.exp(a)\n",
    "print(exp_a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74.1221542101633\n"
     ]
    }
   ],
   "source": [
    "sum_exp_a = np.sum(exp_a)\n",
    "print(sum_exp_a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.5.2 ソフトマックス関数の実装上の注意"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "//anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: RuntimeWarning: invalid value encountered in true_divide\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([1010, 1000, 990])\n",
    "np.exp(a) / np.sum(np.exp(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0, -10, -20])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = np.max(a)\n",
    "a - c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.99954600e-01, 4.53978686e-05, 2.06106005e-09])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.exp(a - c) / np.sum(np.exp(a - c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(a):\n",
    "    c = np.max(a)\n",
    "    exp_a = np.exp(a - c)\n",
    "    sum_exp_a = np.sum(exp_a)\n",
    "    y = exp_a / sum_exp_a\n",
    "    \n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.01821127 0.24519181 0.73659691]\n"
     ]
    }
   ],
   "source": [
    "a = np.array([0.3, 2.9, 4.0])\n",
    "y = softmax(a)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.6.1 MNISTデータセット"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 784)\n",
      "(60000,)\n",
      "(10000, 784)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "import sys, os\n",
    "sys.path.append(os.pardir)\n",
    "from dataset.mnist import load_mnist\n",
    "\n",
    "(x_train, t_train), (x_test, t_test) =\\\n",
    "    load_mnist(flatten=True, normalize=False)\n",
    "\n",
    "# それぞれのデータの形状を出力\n",
    "print(x_train.shape)\n",
    "print(t_train.shape)\n",
    "print(x_test.shape)\n",
    "print(t_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n",
      "(784,)\n",
      "(28, 28)\n"
     ]
    }
   ],
   "source": [
    "import sys, os\n",
    "sys.path.append(os.pardir)\n",
    "import numpy as np\n",
    "from dataset.mnist import load_mnist\n",
    "from PIL import Image\n",
    "\n",
    "def img_show(img):\n",
    "    pil_img = Image.fromarray(np.uint8(img))\n",
    "    pil_img.show()\n",
    "\n",
    "(x_train, t_train), (x_test, t_test) =\\\n",
    "    load_mnist(flatten=True, normalize=False)\n",
    "\n",
    "img = x_train[0]\n",
    "label = t_train[0]\n",
    "print(label)\n",
    "\n",
    "print(img.shape)\n",
    "img = img.reshape(28, 28)\n",
    "print(img.shape)\n",
    "\n",
    "img_show(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data():\n",
    "    (x_train, t_train), (x_test, t_test) =\\\n",
    "        load_mnist(normalize=True, flatten=True, one_hot_label=False)\n",
    "    return x_test, t_test\n",
    "\n",
    "def init_network():\n",
    "    with open(\"ch03/sample_weight.pkl\", 'rb') as f:\n",
    "        network = pickle.load(f)\n",
    "        \n",
    "    return network\n",
    "\n",
    "def predict(network, x):\n",
    "    W1, W2, W3 = network['W1'], network['W2'], network['W3']\n",
    "    b1, b2, b3 = network['b1'], network['b2'], network['b3']\n",
    "    \n",
    "    a1 = np.dot(x, W1) + b1\n",
    "    z1 = sigmoid(a1)\n",
    "    a2 = np.dot(z1, W2) + b2\n",
    "    z2 = sigmoid(a2)\n",
    "    a3 = np.dot(z2, W3) + b3\n",
    "    y = softmax(a3)\n",
    "    \n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:0.9352\n"
     ]
    }
   ],
   "source": [
    "x, t = get_data()\n",
    "network = init_network()\n",
    "\n",
    "accuracy_cnt = 0\n",
    "for i in range(len(x)):\n",
    "    y = predict(network, x[i])\n",
    "    p = np.argmax(y)\n",
    "    \n",
    "    if p == t[i]:\n",
    "        accuracy_cnt += 1\n",
    "        \n",
    "print(\"Accuracy:\" + str(float(accuracy_cnt) / len(x)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.6.3 バッチ処理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 784)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, _ = get_data()\n",
    "network = init_network()\n",
    "W1, W2, W3 = network['W1'], network['W2'], network['W3']\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(784,)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(784, 50)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50, 100)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 10)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:0.9352\n"
     ]
    }
   ],
   "source": [
    "x, t = get_data()\n",
    "network = init_network()\n",
    "\n",
    "batch_size = 100\n",
    "accuracy_cnt = 0\n",
    "\n",
    "for i in range(0, len(x), batch_size):\n",
    "    x_batch = x[i:i+batch_size]\n",
    "    y_batch = predict(network, x_batch)\n",
    "    p = np.argmax(y_batch, axis=1)\n",
    "    accuracy_cnt += np.sum(p == t[i:i+batch_size])\n",
    "\n",
    "print(\"Accuracy:\" + str(float(accuracy_cnt) / len(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.8940939e-08, 8.8548950e-06, 6.9953458e-06, 3.6606650e-06,\n",
       "       8.4337375e-07, 2.5408090e-05, 2.5475965e-06, 1.0850615e-08,\n",
       "       1.4120878e-03, 3.6279488e-07], dtype=float32)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_batch[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2.2 交差エントロピー誤差"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy_error(y, t):\n",
    "    delta = 1e-7\n",
    "    return -np.sum(t * np.log(y + delta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = [0,0,1,0,0,0,0,0,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = [0.1,0.05,0.6,0.0,0.05,0.1,0.0,0.1,0.0,0.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.510825457099338"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_entropy_error(np.array(y), np.array(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = [0.1,0.05,0.1,0.0,0.05,0.1,0.0,0.6,0.0,0.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.302584092994546"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_entropy_error(np.array(y), np.array(t))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2.3 ミニバッチ学習"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 784)\n",
      "(60000, 10)\n"
     ]
    }
   ],
   "source": [
    "import sys, os\n",
    "sys.path.append(os.pardir)\n",
    "import numpy as np\n",
    "from dataset.mnist import load_mnist\n",
    "\n",
    "(x_train, t_train), (x_test, t_test) =\\\n",
    "    load_mnist(normalize=True, one_hot_label=True)\n",
    "\n",
    "print(x_train.shape)\n",
    "print(t_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = x_train.shape[0]\n",
    "batch_size = 10\n",
    "batch_mask = np.random.choice(train_size, batch_size)\n",
    "x_batch = x_train[batch_mask]\n",
    "t_batch = t_train[batch_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 7532,  3337, 43258, 24830, 32122, 44330, 28285, 41276, 43643,\n",
       "       55790])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.choice(60000,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(y, t):\n",
    "    if y.ndim == 1:\n",
    "        t = t.reshape(1, t.size)\n",
    "        y = y.reshape(1, y.size)\n",
    "        \n",
    "    batch_size = y.shape[0]\n",
    "    return -np.sum(t * np.log(y + 1e-7)) / batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}