lab07.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning rate = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 cost:nan, W:[[-0.83333319  0.4166666   0.41666645]\n",
      " [ 1.66666687  2.91666746 -4.58333397]\n",
      " [ 1.66666627  4.16666698 -5.83333397]]\n",
      "20 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "40 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "60 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "80 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "100 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "120 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "140 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "160 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "180 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "200 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "220 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "240 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "260 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "280 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "300 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "320 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "340 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "360 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "380 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "400 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "420 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "440 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "460 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "480 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "500 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "520 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "540 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "560 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "580 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "600 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "620 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "640 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "660 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "680 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "700 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "720 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "740 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "760 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "780 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "800 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "820 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "840 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "860 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "880 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "900 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "920 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "940 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "960 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "980 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1000 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1020 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1040 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1060 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1080 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1100 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1120 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1140 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1160 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1180 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1200 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1220 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1240 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1260 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1280 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1300 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1320 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1340 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1360 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1380 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1400 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1420 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1440 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1460 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1480 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1500 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1520 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1540 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1560 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1580 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1600 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1620 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1640 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1660 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1680 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1700 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1720 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1740 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1760 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1780 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1800 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1820 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1840 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1860 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1880 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1900 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1920 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1940 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1960 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "1980 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "2000 cost:nan, W:[[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]]\n",
      "a : [[ nan  nan  nan]], arg_max(a) : [0]\n",
      "b : [[ nan  nan  nan]], arg_max(c) : [0]\n",
      "c : [[ nan  nan  nan]], arg_max(b) : [0]\n",
      "all_data : [[ nan  nan  nan]\n",
      " [ nan  nan  nan]\n",
      " [ nan  nan  nan]], arg_max(all_data) : [0 0 0]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from IPython.display import display\n",
    "\n",
    "xy = np.loadtxt('data/lab06_train.txt', unpack=True, dtype='float32')\n",
    "x_data = np.transpose(xy[0:3])\n",
    "y_data = np.transpose(xy[3:])\n",
    "\n",
    "X = tf.placeholder('float', [None, 3])\n",
    "Y = tf.placeholder('float', [None, 3])\n",
    "\n",
    "W = tf.Variable(tf.zeros([3,3]))\n",
    "\n",
    "hypothesis = tf.nn.softmax(tf.matmul(X, W))\n",
    "learning_rate = 10\n",
    "cost = tf.reduce_mean(-tf.reduce_sum(Y * tf.log(hypothesis), reduction_indices=1))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for step in range(2001):\n",
    "        sess.run(optimizer, feed_dict={X:x_data, Y:y_data})\n",
    "        if step % 20 == 0:\n",
    "            print(step, \"cost:%s, W:%s\"%(sess.run(cost, feed_dict={X:x_data, Y:y_data}), sess.run(W)))\n",
    "    \n",
    "    a = sess.run(hypothesis, feed_dict={X:[[1,11,7]]})\n",
    "    print(\"a : %s, arg_max(a) : %s\" % (a, sess.run(tf.arg_max(a, 1))))\n",
    "    b = sess.run(hypothesis, feed_dict={X:[[1,3,4]]})\n",
    "    print(\"b : %s, arg_max(c) : %s\" % (b, sess.run(tf.arg_max(b, 1))))\n",
    "    c = sess.run(hypothesis, feed_dict={X:[[1,1,0]]})\n",
    "    print(\"c : %s, arg_max(b) : %s\" % (c, sess.run(tf.arg_max(c, 1))))\n",
    "    all_data = sess.run(hypothesis, feed_dict={X:[[1,11,7], [1,3,4], [1,1,0]]})\n",
    "    print(\"all_data : %s, arg_max(all_data) : %s\" % (all_data, sess.run(tf.arg_max(all_data, 1))))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning rate = 0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 cost:1.09852, W:[[ -8.33333161e-06   4.16666580e-06   4.16666444e-06]\n",
      " [  1.66666687e-05   2.91666747e-05  -4.58333379e-05]\n",
      " [  1.66666614e-05   4.16666662e-05  -5.83333349e-05]]\n",
      "20 cost:1.09683, W:[[ -1.75948357e-04   8.53231104e-05   9.06253117e-05]\n",
      " [  3.45080043e-04   6.01381704e-04  -9.46461631e-04]\n",
      " [  3.45137320e-04   8.63916881e-04  -1.20905391e-03]]\n",
      "40 cost:1.09522, W:[[-0.00034532  0.00016242  0.00018289]\n",
      " [ 0.00066441  0.00115287 -0.00181728]\n",
      " [ 0.00066463  0.0016655  -0.00233013]]\n",
      "60 cost:1.09371, W:[[-0.00051636  0.00023558  0.00028077]\n",
      " [ 0.00097507  0.00168427 -0.00265934]\n",
      " [ 0.00097555  0.00244705 -0.0034226 ]]\n",
      "80 cost:1.09228, W:[[-0.000689    0.00030493  0.00038407]\n",
      " [ 0.00127744  0.0021962  -0.00347363]\n",
      " [ 0.00127827  0.00320918 -0.00448745]]\n",
      "100 cost:1.09092, W:[[-0.00086318  0.00037058  0.0004926 ]\n",
      " [ 0.00157187  0.00268927 -0.00426114]\n",
      " [ 0.00157315  0.00395248 -0.00552563]]\n",
      "120 cost:1.08964, W:[[-0.00103883  0.00043265  0.00060617]\n",
      " [ 0.00185871  0.00316407 -0.00502278]\n",
      " [ 0.00186052  0.00467755 -0.00653806]]\n",
      "140 cost:1.08842, W:[[-0.00121588  0.00049125  0.00072463]\n",
      " [ 0.00213828  0.00362116 -0.00575944]\n",
      " [ 0.00214069  0.00538493 -0.00752562]]\n",
      "160 cost:1.08727, W:[[-0.00139429  0.00054648  0.00084781]\n",
      " [ 0.00241088  0.0040611  -0.00647198]\n",
      " [ 0.00241398  0.00607517 -0.00848915]]\n",
      "180 cost:1.08618, W:[[-0.001574    0.00059845  0.00097555]\n",
      " [ 0.00267679  0.00448442 -0.00716121]\n",
      " [ 0.00268065  0.00674881 -0.00942946]]\n",
      "200 cost:1.08514, W:[[-0.00175495  0.00064726  0.00110769]\n",
      " [ 0.00293629  0.00489164 -0.00782793]\n",
      " [ 0.00294097  0.00740635 -0.01034732]]\n",
      "220 cost:1.08416, W:[[-0.0019371   0.00069301  0.00124409]\n",
      " [ 0.00318962  0.00528326 -0.00847288]\n",
      " [ 0.00319519  0.0080483  -0.01124349]]\n",
      "240 cost:1.08322, W:[[-0.00212041  0.00073579  0.00138462]\n",
      " [ 0.00343703  0.00565976 -0.00909679]\n",
      " [ 0.00344355  0.00867512 -0.01211867]]\n",
      "260 cost:1.08234, W:[[-0.00230482  0.00077569  0.00152913]\n",
      " [ 0.00367874  0.00602163 -0.00970037]\n",
      " [ 0.00368627  0.00928729 -0.01297356]]\n",
      "280 cost:1.08149, W:[[-0.0024903   0.0008128   0.0016775 ]\n",
      " [ 0.00391497  0.00636931 -0.01028428]\n",
      " [ 0.00392357  0.00988525 -0.01380882]]\n",
      "300 cost:1.08069, W:[[-0.00267681  0.00084721  0.00182961]\n",
      " [ 0.00414591  0.00670326 -0.01084916]\n",
      " [ 0.00415563  0.01046945 -0.01462509]]\n",
      "320 cost:1.07993, W:[[-0.00286432  0.000879    0.00198532]\n",
      " [ 0.00437175  0.00702389 -0.01139564]\n",
      " [ 0.00438266  0.01104031 -0.01542297]]\n",
      "340 cost:1.0792, W:[[-0.00305278  0.00090825  0.00214453]\n",
      " [ 0.00459268  0.00733162 -0.0119243 ]\n",
      " [ 0.00460482  0.01159825 -0.01620307]]\n",
      "360 cost:1.07851, W:[[-0.00324217  0.00093504  0.00230713]\n",
      " [ 0.00480887  0.00762686 -0.01243573]\n",
      " [ 0.00482229  0.01214364 -0.01696594]]\n",
      "380 cost:1.07785, W:[[-0.00343245  0.00095944  0.00247301]\n",
      " [ 0.00502048  0.00791    -0.01293048]\n",
      " [ 0.00503523  0.01267689 -0.01771213]]\n",
      "400 cost:1.07722, W:[[-0.0036236   0.00098153  0.00264207]\n",
      " [ 0.00522766  0.00818141 -0.01340907]\n",
      " [ 0.00524379  0.01319837 -0.01844216]]\n",
      "420 cost:1.07662, W:[[-0.00381559  0.00100139  0.0028142 ]\n",
      " [ 0.00543056  0.00844145 -0.01387201]\n",
      " [ 0.00544811  0.01370843 -0.01915654]]\n",
      "440 cost:1.07605, W:[[-0.00400838  0.00101906  0.00298932]\n",
      " [ 0.00562931  0.00869049 -0.01431981]\n",
      " [ 0.00564834  0.01420742 -0.01985575]]\n",
      "460 cost:1.0755, W:[[-0.00420196  0.00103464  0.00316732]\n",
      " [ 0.00582406  0.00892886 -0.01475292]\n",
      " [ 0.00584459  0.01469569 -0.02054028]]\n",
      "480 cost:1.07497, W:[[-0.0043963   0.00104817  0.00334813]\n",
      " [ 0.00601491  0.00915689 -0.01517181]\n",
      " [ 0.006037    0.01517356 -0.02121055]]\n",
      "500 cost:1.07447, W:[[-0.00459138  0.00105972  0.00353165]\n",
      " [ 0.00620199  0.00937491 -0.01557691]\n",
      " [ 0.00622568  0.01564134 -0.02186702]]\n",
      "520 cost:1.07399, W:[[-0.00478717  0.00106936  0.00371781]\n",
      " [ 0.00638542  0.00958323 -0.01596866]\n",
      " [ 0.00641074  0.01609936 -0.02251009]]\n",
      "540 cost:1.07353, W:[[-0.00498365  0.00107713  0.00390652]\n",
      " [ 0.0065653   0.00978214 -0.01634745]\n",
      " [ 0.0065923   0.01654789 -0.02314017]]\n",
      "560 cost:1.07309, W:[[-0.00518081  0.0010831   0.00409771]\n",
      " [ 0.00674173  0.00997194 -0.01671368]\n",
      " [ 0.00677045  0.01698723 -0.02375767]]\n",
      "580 cost:1.07267, W:[[-0.00537862  0.00108732  0.0042913 ]\n",
      " [ 0.00691481  0.01015291 -0.01706773]\n",
      " [ 0.00694528  0.01741767 -0.02436294]]\n",
      "600 cost:1.07226, W:[[-0.00557707  0.00108985  0.00448722]\n",
      " [ 0.00708463  0.01032533 -0.01740997]\n",
      " [ 0.0071169   0.01783946 -0.02495635]]\n",
      "620 cost:1.07187, W:[[-0.00577614  0.00109073  0.00468541]\n",
      " [ 0.00725129  0.01048945 -0.01774075]\n",
      " [ 0.00728539  0.01825287 -0.02553824]]\n",
      "640 cost:1.0715, W:[[-0.0059758   0.00109001  0.00488579]\n",
      " [ 0.00741486  0.01064554 -0.01806041]\n",
      " [ 0.00745084  0.01865815 -0.02610897]]\n",
      "660 cost:1.07113, W:[[-0.00617606  0.00108776  0.0050883 ]\n",
      " [ 0.00757544  0.01079384 -0.01836928]\n",
      " [ 0.00761332  0.01905555 -0.02666885]]\n",
      "680 cost:1.07079, W:[[-0.00637688  0.001084    0.00529289]\n",
      " [ 0.00773309  0.01093458 -0.01866768]\n",
      " [ 0.00777291  0.01944531 -0.0272182 ]]\n",
      "700 cost:1.07045, W:[[-0.00657826  0.00107878  0.00549948]\n",
      " [ 0.00788789  0.01106801 -0.01895591]\n",
      " [ 0.0079297   0.01982765 -0.02775732]]\n",
      "720 cost:1.07013, W:[[-0.00678018  0.00107216  0.00570803]\n",
      " [ 0.00803992  0.01119435 -0.01923428]\n",
      " [ 0.00808375  0.0202028  -0.02828651]]\n",
      "740 cost:1.06982, W:[[-0.00698263  0.00106417  0.00591847]\n",
      " [ 0.00818924  0.01131382 -0.01950306]\n",
      " [ 0.00823512  0.02057097 -0.02880606]]\n",
      "760 cost:1.06952, W:[[-0.00718559  0.00105484  0.00613075]\n",
      " [ 0.00833592  0.01142661 -0.01976253]\n",
      " [ 0.00838389  0.02093236 -0.02931622]]\n",
      "780 cost:1.06923, W:[[-0.00738905  0.00104424  0.00634482]\n",
      " [ 0.00848002  0.01153295 -0.02001297]\n",
      " [ 0.00853012  0.02128719 -0.02981728]]\n",
      "800 cost:1.06895, W:[[-0.00759301  0.00103238  0.00656063]\n",
      " [ 0.00862161  0.01163302 -0.02025463]\n",
      " [ 0.00867387  0.02163565 -0.03030949]]\n",
      "820 cost:1.06868, W:[[-0.00779743  0.00101931  0.00677812]\n",
      " [ 0.00876074  0.01172702 -0.02048776]\n",
      " [ 0.0088152   0.02197792 -0.03079309]]\n",
      "840 cost:1.06841, W:[[-0.00800233  0.00100507  0.00699726]\n",
      " [ 0.00889747  0.01181513 -0.0207126 ]\n",
      " [ 0.00895416  0.02231419 -0.03126832]]\n",
      "860 cost:1.06816, W:[[-0.00820768  0.00098968  0.00721799]\n",
      " [ 0.00903186  0.01189753 -0.02092939]\n",
      " [ 0.00909082  0.02264464 -0.03173542]]\n",
      "880 cost:1.06791, W:[[-0.00841347  0.00097319  0.00744028]\n",
      " [ 0.00916395  0.01197439 -0.02113835]\n",
      " [ 0.00922521  0.02296944 -0.03219462]]\n",
      "900 cost:1.06767, W:[[-0.00861969  0.00095562  0.00766407]\n",
      " [ 0.0092938   0.01204589 -0.02133971]\n",
      " [ 0.00935741  0.02328875 -0.03264612]]\n",
      "920 cost:1.06744, W:[[-0.00882634  0.00093701  0.00788932]\n",
      " [ 0.00942146  0.01211219 -0.02153367]\n",
      " [ 0.00948744  0.02360274 -0.03309015]]\n",
      "940 cost:1.06722, W:[[-0.00903341  0.00091739  0.00811601]\n",
      " [ 0.00954698  0.01217344 -0.02172044]\n",
      " [ 0.00961537  0.02391155 -0.0335269 ]]\n",
      "960 cost:1.067, W:[[-0.00924087  0.00089679  0.00834408]\n",
      " [ 0.0096704   0.0122298  -0.02190021]\n",
      " [ 0.00974124  0.02421536 -0.03395656]]\n",
      "980 cost:1.06678, W:[[-0.00944873  0.00087524  0.00857349]\n",
      " [ 0.00979177  0.01228141 -0.0220732 ]\n",
      " [ 0.00986509  0.02451428 -0.03437934]]\n",
      "1000 cost:1.06658, W:[[-0.00965698  0.00085276  0.00880422]\n",
      " [ 0.00991114  0.01232841 -0.02223956]\n",
      " [ 0.00998697  0.02480848 -0.03479542]]\n",
      "1020 cost:1.06637, W:[[-0.00986561  0.00082938  0.00903622]\n",
      " [ 0.01002853  0.01237095 -0.0223995 ]\n",
      " [ 0.01010691  0.02509809 -0.03520497]]\n",
      "1040 cost:1.06618, W:[[-0.01007461  0.00080513  0.00926946]\n",
      " [ 0.010144    0.01240916 -0.02255318]\n",
      " [ 0.01022497  0.02538325 -0.03560818]]\n",
      "1060 cost:1.06598, W:[[-0.01028396  0.00078004  0.00950391]\n",
      " [ 0.01025759  0.01244317 -0.02270078]\n",
      " [ 0.01034117  0.02566406 -0.0360052 ]]\n",
      "1080 cost:1.06579, W:[[-0.01049367  0.00075412  0.00973954]\n",
      " [ 0.01036933  0.01247311 -0.02284245]\n",
      " [ 0.01045557  0.02594068 -0.03639621]]\n",
      "1100 cost:1.06561, W:[[-0.01070372  0.00072741  0.00997631]\n",
      " [ 0.01047925  0.01249909 -0.02297836]\n",
      " [ 0.01056819  0.0262132  -0.03678135]]\n",
      "1120 cost:1.06543, W:[[-0.01091412  0.00069992  0.01021419]\n",
      " [ 0.01058741  0.01252123 -0.02310867]\n",
      " [ 0.01067907  0.02648176 -0.0371608 ]]\n",
      "1140 cost:1.06526, W:[[-0.01112484  0.00067168  0.01045316]\n",
      " [ 0.01069383  0.01253966 -0.02323352]\n",
      " [ 0.01078825  0.02674647 -0.03753468]]\n",
      "1160 cost:1.06508, W:[[-0.01133589  0.00064271  0.01069317]\n",
      " [ 0.01079855  0.01255448 -0.02335305]\n",
      " [ 0.01089577  0.02700743 -0.03790316]]\n",
      "1180 cost:1.06492, W:[[-0.01154725  0.00061303  0.01093422]\n",
      " [ 0.0109016   0.01256579 -0.02346742]\n",
      " [ 0.01100165  0.02726475 -0.03826638]]\n",
      "1200 cost:1.06475, W:[[-0.01175893  0.00058266  0.01117626]\n",
      " [ 0.01100301  0.01257371 -0.02357675]\n",
      " [ 0.01110593  0.02751854 -0.03862444]]\n",
      "1220 cost:1.06459, W:[[-0.0119709   0.00055162  0.01141928]\n",
      " [ 0.01110282  0.01257834 -0.02368118]\n",
      " [ 0.01120864  0.02776889 -0.03897751]]\n",
      "1240 cost:1.06443, W:[[-0.01218318  0.00051993  0.01166324]\n",
      " [ 0.01120106  0.01257976 -0.02378085]\n",
      " [ 0.01130982  0.02801591 -0.0393257 ]]\n",
      "1260 cost:1.06427, W:[[-0.01239574  0.00048761  0.01190812]\n",
      " [ 0.01129776  0.01257809 -0.02387587]\n",
      " [ 0.01140949  0.02825968 -0.03966914]]\n",
      "1280 cost:1.06412, W:[[-0.01260859  0.00045468  0.0121539 ]\n",
      " [ 0.01139295  0.01257339 -0.02396636]\n",
      " [ 0.01150768  0.0285003  -0.04000796]]\n",
      "1300 cost:1.06397, W:[[-0.01282172  0.00042115  0.01240056]\n",
      " [ 0.01148665  0.01256578 -0.02405245]\n",
      " [ 0.01160443  0.02873787 -0.04034226]]\n",
      "1320 cost:1.06382, W:[[-0.01303512  0.00038705  0.01264807]\n",
      " [ 0.0115789   0.01255533 -0.02413425]\n",
      " [ 0.01169976  0.02897245 -0.04067216]]\n",
      "1340 cost:1.06368, W:[[-0.01324879  0.00035238  0.0128964 ]\n",
      " [ 0.01166973  0.01254213 -0.02421188]\n",
      " [ 0.0117937   0.02920414 -0.04099779]]\n",
      "1360 cost:1.06353, W:[[-0.01346272  0.00031716  0.01314555]\n",
      " [ 0.01175916  0.01252626 -0.02428543]\n",
      " [ 0.01188628  0.029433   -0.04131923]]\n",
      "1380 cost:1.06339, W:[[-0.0136769   0.00028142  0.01339548]\n",
      " [ 0.01184722  0.01250779 -0.02435502]\n",
      " [ 0.01197752  0.02965913 -0.0416366 ]]\n",
      "1400 cost:1.06325, W:[[-0.01389133  0.00024516  0.01364617]\n",
      " [ 0.01193393  0.0124868  -0.02442075]\n",
      " [ 0.01206745  0.0298826  -0.04195001]]\n",
      "1420 cost:1.06311, W:[[-0.01410601  0.00020839  0.01389761]\n",
      " [ 0.01201932  0.01246338 -0.02448272]\n",
      " [ 0.0121561   0.03010348 -0.04225954]]\n",
      "1440 cost:1.06298, W:[[-0.01432093  0.00017114  0.01414978]\n",
      " [ 0.01210342  0.01243758 -0.02454102]\n",
      " [ 0.01224349  0.03032185 -0.04256528]]\n",
      "1460 cost:1.06285, W:[[-0.01453609  0.00013342  0.01440265]\n",
      " [ 0.01218625  0.01240948 -0.02459574]\n",
      " [ 0.01232964  0.03053776 -0.04286735]]\n",
      "1480 cost:1.06271, W:[[ -1.47514688e-02   9.52338305e-05   1.46562234e-02]\n",
      " [  1.22678299e-02   1.23791425e-02  -2.46469919e-02]\n",
      " [  1.24145821e-02   3.07512954e-02  -4.31658290e-02]]\n",
      "1500 cost:1.06258, W:[[ -1.49670774e-02   5.66016352e-05   1.49104632e-02]\n",
      " [  1.23481890e-02   1.23466374e-02  -2.46948451e-02]\n",
      " [  1.24983378e-02   3.09625138e-02  -4.34608012e-02]]\n",
      "1520 cost:1.06245, W:[[ -1.51829068e-02   1.75343594e-05   1.51653597e-02]\n",
      " [  1.24273496e-02   1.23120267e-02  -2.47393940e-02]\n",
      " [  1.25809284e-02   3.11714839e-02  -4.37523536e-02]]\n",
      "1540 cost:1.06233, W:[[ -1.53989512e-02  -2.19558642e-05   1.54208941e-02]\n",
      " [  1.25053320e-02   1.22753717e-02  -2.47807205e-02]\n",
      " [  1.26623763e-02   3.13782543e-02  -4.40405756e-02]]\n",
      "1560 cost:1.0622, W:[[ -1.56152062e-02  -6.18573249e-05   1.56770516e-02]\n",
      " [  1.25821615e-02   1.22367321e-02  -2.48189103e-02]\n",
      " [  1.27427038e-02   3.15828919e-02  -4.43255380e-02]]\n",
      "1580 cost:1.06208, W:[[-0.01583167 -0.00010216  0.01593381]\n",
      " [ 0.01265785  0.01219617 -0.02485404]\n",
      " [ 0.01282194  0.03178545 -0.04460733]]\n",
      "1600 cost:1.06195, W:[[-0.01604833 -0.00014285  0.01619117]\n",
      " [ 0.01273244  0.01215373 -0.02488619]\n",
      " [ 0.01290009  0.031986   -0.04488603]]\n",
      "1620 cost:1.06183, W:[[-0.01626519 -0.00018392  0.0164491 ]\n",
      " [ 0.01280593  0.01210947 -0.02491543]\n",
      " [ 0.01297719  0.03218457 -0.0451617 ]]\n",
      "1640 cost:1.06171, W:[[-0.01648225 -0.00022535  0.01670758]\n",
      " [ 0.01287835  0.01206345 -0.02494184]\n",
      " [ 0.01305326  0.03238123 -0.04543442]]\n",
      "1660 cost:1.06159, W:[[-0.01669949 -0.00026715  0.01696662]\n",
      " [ 0.01294972  0.01201573 -0.02496548]\n",
      " [ 0.01312831  0.03257603 -0.04570426]]\n",
      "1680 cost:1.06147, W:[[-0.01691691 -0.00030929  0.01722619]\n",
      " [ 0.01302007  0.01196633 -0.02498643]\n",
      " [ 0.01320237  0.03276902 -0.0459713 ]]\n",
      "1700 cost:1.06135, W:[[-0.01713452 -0.00035177  0.01748628]\n",
      " [ 0.0130894   0.01191532 -0.02500475]\n",
      " [ 0.01327544  0.03296024 -0.04623559]]\n",
      "1720 cost:1.06123, W:[[-0.01735231 -0.00039457  0.01774688]\n",
      " [ 0.01315774  0.01186275 -0.02502051]\n",
      " [ 0.01334757  0.03314974 -0.04649721]]\n",
      "1740 cost:1.06112, W:[[-0.01757028 -0.0004377   0.01800797]\n",
      " [ 0.0132251   0.01180864 -0.02503377]\n",
      " [ 0.01341875  0.03333756 -0.04675622]]\n",
      "1760 cost:1.061, W:[[-0.01778841 -0.00048114  0.01826954]\n",
      " [ 0.01329151  0.01175307 -0.02504461]\n",
      " [ 0.01348901  0.03352375 -0.04701268]]\n",
      "1780 cost:1.06089, W:[[-0.01800671 -0.00052488  0.01853158]\n",
      " [ 0.01335698  0.01169605 -0.02505306]\n",
      " [ 0.01355837  0.03370835 -0.04726664]]\n",
      "1800 cost:1.06077, W:[[-0.01822517 -0.00056892  0.01879408]\n",
      " [ 0.01342153  0.01163764 -0.02505921]\n",
      " [ 0.01362685  0.0338914  -0.04751818]]\n",
      "1820 cost:1.06066, W:[[-0.01844379 -0.00061325  0.01905702]\n",
      " [ 0.01348518  0.01157788 -0.0250631 ]\n",
      " [ 0.01369446  0.03407294 -0.04776733]]\n",
      "1840 cost:1.06055, W:[[-0.01866256 -0.00065785  0.0193204 ]\n",
      " [ 0.01354795  0.0115168  -0.02506479]\n",
      " [ 0.01376121  0.03425302 -0.04801418]]\n",
      "1860 cost:1.06044, W:[[-0.01888149 -0.00070272  0.0195842 ]\n",
      " [ 0.01360984  0.01145445 -0.02506432]\n",
      " [ 0.01382714  0.03443167 -0.04825874]]\n",
      "1880 cost:1.06033, W:[[-0.01910056 -0.00074786  0.01984842]\n",
      " [ 0.01367088  0.01139085 -0.02506177]\n",
      " [ 0.01389224  0.03460892 -0.0485011 ]]\n",
      "1900 cost:1.06021, W:[[-0.01931978 -0.00079326  0.02011303]\n",
      " [ 0.01373109  0.01132605 -0.02505718]\n",
      " [ 0.01395655  0.03478482 -0.04874131]]\n",
      "1920 cost:1.06011, W:[[-0.01953915 -0.0008389   0.02037804]\n",
      " [ 0.01379048  0.01126008 -0.02505059]\n",
      " [ 0.01402006  0.0349594  -0.04897941]]\n",
      "1940 cost:1.06, W:[[-0.01975865 -0.00088479  0.02064343]\n",
      " [ 0.01384905  0.01119298 -0.02504206]\n",
      " [ 0.01408281  0.03513269 -0.04921544]]\n",
      "1960 cost:1.05989, W:[[-0.01997828 -0.00093092  0.0209092 ]\n",
      " [ 0.01390684  0.01112477 -0.02503164]\n",
      " [ 0.0141448   0.03530472 -0.04944947]]\n",
      "1980 cost:1.05978, W:[[-0.02019806 -0.00097727  0.02117532]\n",
      " [ 0.01396386  0.01105549 -0.02501938]\n",
      " [ 0.01420605  0.03547552 -0.04968152]]\n",
      "2000 cost:1.05967, W:[[-0.02041796 -0.00102386  0.02144181]\n",
      " [ 0.01402011  0.01098517 -0.02500531]\n",
      " [ 0.01426658  0.03564513 -0.04991166]]\n",
      "a : [[ 0.38783085  0.44417956  0.16798955]], arg_max(a) : [1]\n",
      "b : [[ 0.35485148  0.39052922  0.25461939]], arg_max(c) : [1]\n",
      "c : [[ 0.33119905  0.33666176  0.33213913]], arg_max(b) : [1]\n",
      "all_data : [[ 0.38783085  0.44417956  0.16798955]\n",
      " [ 0.35485148  0.39052922  0.25461939]\n",
      " [ 0.33119905  0.33666176  0.33213913]], arg_max(all_data) : [1 1 1]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from IPython.display import display\n",
    "\n",
    "xy = np.loadtxt('data/lab06_train.txt', unpack=True, dtype='float32')\n",
    "x_data = np.transpose(xy[0:3])\n",
    "y_data = np.transpose(xy[3:])\n",
    "\n",
    "X = tf.placeholder('float', [None, 3])\n",
    "Y = tf.placeholder('float', [None, 3])\n",
    "\n",
    "W = tf.Variable(tf.zeros([3,3]))\n",
    "\n",
    "hypothesis = tf.nn.softmax(tf.matmul(X, W))\n",
    "learning_rate = 0.0001\n",
    "cost = tf.reduce_mean(-tf.reduce_sum(Y * tf.log(hypothesis), reduction_indices=1))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for step in range(2001):\n",
    "        sess.run(optimizer, feed_dict={X:x_data, Y:y_data})\n",
    "        if step % 20 == 0:\n",
    "            print(step, \"cost:%s, W:%s\"%(sess.run(cost, feed_dict={X:x_data, Y:y_data}), sess.run(W)))\n",
    "    \n",
    "    a = sess.run(hypothesis, feed_dict={X:[[1,11,7]]})\n",
    "    print(\"a : %s, arg_max(a) : %s\" % (a, sess.run(tf.arg_max(a, 1))))\n",
    "    b = sess.run(hypothesis, feed_dict={X:[[1,3,4]]})\n",
    "    print(\"b : %s, arg_max(c) : %s\" % (b, sess.run(tf.arg_max(b, 1))))\n",
    "    c = sess.run(hypothesis, feed_dict={X:[[1,1,0]]})\n",
    "    print(\"c : %s, arg_max(b) : %s\" % (c, sess.run(tf.arg_max(c, 1))))\n",
    "    all_data = sess.run(hypothesis, feed_dict={X:[[1,11,7], [1,3,4], [1,1,0]]})\n",
    "    print(\"all_data : %s, arg_max(all_data) : %s\" % (all_data, sess.run(tf.arg_max(all_data, 1))))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 cost:1.09048, W:[[-0.00083333  0.00041667  0.00041667]\n",
      " [ 0.00166667  0.00291667 -0.00458333]\n",
      " [ 0.00166667  0.00416667 -0.00583333]]\n",
      "20 cost:1.05909, W:[[-0.02147574 -0.00120662  0.02268235]\n",
      " [ 0.01449275  0.01086621 -0.02535895]\n",
      " [ 0.0147527   0.03674991 -0.05150261]]\n",
      "40 cost:1.04942, W:[[-0.04383909 -0.0064992   0.05033828]\n",
      " [ 0.01761899  0.00110689 -0.01872586]\n",
      " [ 0.01848182  0.05016028 -0.06864209]]\n",
      "60 cost:1.04029, W:[[-0.06619629 -0.01201317  0.07820947]\n",
      " [ 0.0195295  -0.00897234 -0.01055714]\n",
      " [ 0.02128116  0.06173807 -0.08301921]]\n",
      "80 cost:1.03154, W:[[-0.08835386 -0.01754473  0.10589862]\n",
      " [ 0.02120088 -0.01843084 -0.00277002]\n",
      " [ 0.02408877  0.07251359 -0.09660233]]\n",
      "100 cost:1.02315, W:[[-0.11029428 -0.02307285  0.13336717]\n",
      " [ 0.02274159 -0.02723114  0.00448957]\n",
      " [ 0.02698387  0.08261414 -0.10959798]]\n",
      "120 cost:1.01509, W:[[-0.13201785 -0.02859196  0.16060989]\n",
      " [ 0.02417225 -0.03540835  0.01123613]\n",
      " [ 0.02996077  0.09208993 -0.12205065]]\n",
      "140 cost:1.00734, W:[[-0.15352683 -0.03409813  0.18762501]\n",
      " [ 0.02550456 -0.0430021   0.01749758]\n",
      " [ 0.03300676  0.10098173 -0.13398844]]\n",
      "160 cost:0.999856, W:[[-0.17482381 -0.03958778  0.2144116 ]\n",
      " [ 0.02674878 -0.05005032  0.02330158]\n",
      " [ 0.03610953  0.1093272  -0.14543667]]\n",
      "180 cost:0.992635, W:[[-0.19591154 -0.04505755  0.24096914]\n",
      " [ 0.02791442 -0.05658866  0.02867429]\n",
      " [ 0.03925773  0.11716156 -0.15641922]]\n",
      "200 cost:0.985653, W:[[-0.21679303 -0.05050437  0.26729742]\n",
      " [ 0.02901031 -0.06265054  0.03364029]\n",
      " [ 0.04244109  0.12451769 -0.16695869]]\n",
      "220 cost:0.978895, W:[[-0.23747128 -0.05592537  0.29339662]\n",
      " [ 0.0300446  -0.06826727  0.03822274]\n",
      " [ 0.04565033  0.13142633 -0.17707655]]\n",
      "240 cost:0.972345, W:[[-0.25794938 -0.06131788  0.31926727]\n",
      " [ 0.03102476 -0.07346807  0.04244338]\n",
      " [ 0.04887707  0.13791612 -0.18679309]]\n",
      "260 cost:0.965989, W:[[-0.27823052 -0.06667949  0.34491003]\n",
      " [ 0.03195766 -0.07828024  0.04632268]\n",
      " [ 0.05211383  0.14401382 -0.19612755]]\n",
      "280 cost:0.959817, W:[[-0.29831791 -0.07200795  0.37032577]\n",
      " [ 0.03284956 -0.08272932  0.04987987]\n",
      " [ 0.05535392  0.14974436 -0.20509809]]\n",
      "300 cost:0.953816, W:[[-0.3182148  -0.07730121  0.39551598]\n",
      " [ 0.03370612 -0.08683903  0.05313304]\n",
      " [ 0.0585914   0.15513091 -0.21372208]]\n",
      "320 cost:0.947977, W:[[-0.33792448 -0.08255742  0.42048183]\n",
      " [ 0.03453247 -0.09063158  0.05609922]\n",
      " [ 0.061821    0.16019514 -0.22201593]]\n",
      "340 cost:0.942291, W:[[-0.3574501  -0.0877749   0.44522488]\n",
      " [ 0.03533328 -0.09412765  0.05879451]\n",
      " [ 0.06503811  0.16495728 -0.22999515]]\n",
      "360 cost:0.93675, W:[[-0.37679502 -0.09295209  0.46974692]\n",
      " [ 0.03611267 -0.09734651  0.06123397]\n",
      " [ 0.06823869  0.16943607 -0.23767447]]\n",
      "380 cost:0.931346, W:[[-0.39596227 -0.09808763  0.49404973]\n",
      " [ 0.03687439 -0.10030617  0.0634319 ]\n",
      " [ 0.07141922  0.17364909 -0.24506801]]\n",
      "400 cost:0.926073, W:[[-0.41495511 -0.10318027  0.51813519]\n",
      " [ 0.03762176 -0.10302337  0.06540174]\n",
      " [ 0.07457665  0.17761268 -0.25218898]]\n",
      "420 cost:0.920923, W:[[-0.43377677 -0.10822889  0.54200524]\n",
      " [ 0.03835771 -0.10551376  0.06715621]\n",
      " [ 0.07770839  0.18134212 -0.25905013]]\n",
      "440 cost:0.915892, W:[[-0.45243025 -0.1132325   0.56566244]\n",
      " [ 0.03908487 -0.10779194  0.06870726]\n",
      " [ 0.08081225  0.18485165 -0.26566359]]\n",
      "460 cost:0.910974, W:[[-0.47091874 -0.11819023  0.58910865]\n",
      " [ 0.03980554 -0.10987155  0.07006622]\n",
      " [ 0.08388636  0.18815459 -0.27204061]]\n",
      "480 cost:0.906165, W:[[-0.48924524 -0.12310128  0.61234623]\n",
      " [ 0.04052173 -0.11176533  0.07124381]\n",
      " [ 0.08692922  0.19126342 -0.27819228]]\n",
      "500 cost:0.901458, W:[[-0.50741267 -0.127965    0.63537741]\n",
      " [ 0.04123517 -0.11348514  0.0722502 ]\n",
      " [ 0.08993956  0.19418971 -0.28412896]]\n",
      "520 cost:0.896851, W:[[-0.52542406 -0.13278078  0.6582045 ]\n",
      " [ 0.04194738 -0.11504209  0.07309496]\n",
      " [ 0.09291644  0.19694437 -0.28986046]]\n",
      "540 cost:0.89234, W:[[-0.54328215 -0.13754809  0.68083   ]\n",
      " [ 0.04265967 -0.11644656  0.07378716]\n",
      " [ 0.09585912  0.19953759 -0.2953963 ]]\n",
      "560 cost:0.88792, W:[[-0.56098992 -0.14226651  0.70325607]\n",
      " [ 0.04337312 -0.11770826  0.07433544]\n",
      " [ 0.09876705  0.20197886 -0.30074549]]\n",
      "580 cost:0.883588, W:[[-0.57855004 -0.14693569  0.72548527]\n",
      " [ 0.04408867 -0.11883629  0.07474793]\n",
      " [ 0.10163989  0.20427717 -0.30591658]]\n",
      "600 cost:0.879342, W:[[-0.59596533 -0.1515553   0.74752003]\n",
      " [ 0.04480708 -0.11983915  0.07503238]\n",
      " [ 0.10447746  0.20644082 -0.31091776]]\n",
      "620 cost:0.875177, W:[[-0.61323822 -0.15612513  0.76936275]\n",
      " [ 0.04552896 -0.12072476  0.07519612]\n",
      " [ 0.10727972  0.20847768 -0.31575686]]\n",
      "640 cost:0.871092, W:[[-0.63037139 -0.16064499  0.7910158 ]\n",
      " [ 0.0462548  -0.12150063  0.07524613]\n",
      " [ 0.11004674  0.21039513 -0.3204414 ]]\n",
      "660 cost:0.867083, W:[[-0.64736748 -0.16511475  0.81248158]\n",
      " [ 0.04698502 -0.12217371  0.07518899]\n",
      " [ 0.11277876  0.21220006 -0.32497829]]\n",
      "680 cost:0.863149, W:[[-0.66422892 -0.16953433  0.83376253]\n",
      " [ 0.04771988 -0.12275057  0.07503102]\n",
      " [ 0.11547608  0.21389897 -0.32937449]]\n",
      "700 cost:0.859286, W:[[-0.68095815 -0.17390366  0.85486108]\n",
      " [ 0.04845956 -0.12323735  0.07477812]\n",
      " [ 0.11813907  0.21549796 -0.33363649]]\n",
      "720 cost:0.855492, W:[[-0.69755757 -0.17822278  0.87577957]\n",
      " [ 0.04920416 -0.12363978  0.07443598]\n",
      " [ 0.12076823  0.21700272 -0.3377704 ]]\n",
      "740 cost:0.851766, W:[[-0.71402937 -0.18249169  0.89652038]\n",
      " [ 0.04995373 -0.12396329  0.07400996]\n",
      " [ 0.12336407  0.21841866 -0.34178215]]\n",
      "760 cost:0.848105, W:[[-0.73037601 -0.18671051  0.91708583]\n",
      " [ 0.05070821 -0.12421295  0.07350516]\n",
      " [ 0.12592719  0.21975085 -0.3456775 ]]\n",
      "780 cost:0.844507, W:[[-0.74659961 -0.19087929  0.9374783 ]\n",
      " [ 0.05146754 -0.12439357  0.07292645]\n",
      " [ 0.1284582   0.22100405 -0.34946173]]\n",
      "800 cost:0.840971, W:[[-0.76270223 -0.19499816  0.95770001]\n",
      " [ 0.05223157 -0.12450957  0.07227841]\n",
      " [ 0.13095778  0.22218271 -0.35314   ]]\n",
      "820 cost:0.837495, W:[[-0.77868634 -0.19906732  0.97775304]\n",
      " [ 0.05300014 -0.12456517  0.07156543]\n",
      " [ 0.13342664  0.2232911  -0.35671723]]\n",
      "840 cost:0.834077, W:[[-0.79455376 -0.20308691  0.99764007]\n",
      " [ 0.05377305 -0.12456433  0.07079168]\n",
      " [ 0.13586552  0.22433318 -0.36019817]]\n",
      "860 cost:0.830715, W:[[-0.81030649 -0.20705718  1.01736307]\n",
      " [ 0.05455005 -0.12451074  0.06996112]\n",
      " [ 0.13827516  0.22531272 -0.36358729]]\n",
      "880 cost:0.827408, W:[[-0.82594651 -0.2109783   1.03692424]\n",
      " [ 0.05533086 -0.12440792  0.0690775 ]\n",
      " [ 0.14065629  0.22623317 -0.36688891]]\n",
      "900 cost:0.824155, W:[[-0.84147584 -0.21485054  1.05632591]\n",
      " [ 0.0561152  -0.12425917  0.06814439]\n",
      " [ 0.14300969  0.22709794 -0.37010711]]\n",
      "920 cost:0.820954, W:[[-0.85689634 -0.21867417  1.07556999]\n",
      " [ 0.05690278 -0.12406755  0.06716518]\n",
      " [ 0.14533617  0.22791012 -0.37324575]]\n",
      "940 cost:0.817805, W:[[-0.87220979 -0.22244941  1.09465873]\n",
      " [ 0.05769325 -0.12383595  0.06614308]\n",
      " [ 0.14763647  0.22867264 -0.37630856]]\n",
      "960 cost:0.814704, W:[[-0.88741797 -0.22617656  1.11359406]\n",
      " [ 0.05848633 -0.12356712  0.06508117]\n",
      " [ 0.14991139  0.22938834 -0.37929919]]\n",
      "980 cost:0.811652, W:[[-0.90252274 -0.22985595  1.13237822]\n",
      " [ 0.05928163 -0.12326363  0.06398238]\n",
      " [ 0.1521617   0.2300598  -0.38222101]]\n",
      "1000 cost:0.808647, W:[[-0.91752577 -0.2334879   1.15101326]\n",
      " [ 0.06007884 -0.12292791  0.06284945]\n",
      " [ 0.15438823  0.23068959 -0.38507724]]\n",
      "1020 cost:0.805688, W:[[-0.93242872 -0.23707274  1.16950095]\n",
      " [ 0.0608776  -0.12256224  0.06168501]\n",
      " [ 0.15659165  0.23127997 -0.38787103]]\n",
      "1040 cost:0.802774, W:[[-0.9472332  -0.24061076  1.18784368]\n",
      " [ 0.06167758 -0.12216874  0.06049155]\n",
      " [ 0.15877277  0.23183317 -0.39060533]]\n",
      "1060 cost:0.799904, W:[[-0.96194124 -0.24410234  1.20604336]\n",
      " [ 0.06247844 -0.12174944  0.05927141]\n",
      " [ 0.16093233  0.23235129 -0.39328301]]\n",
      "1080 cost:0.797077, W:[[-0.97655392 -0.24754781  1.22410119]\n",
      " [ 0.06327983 -0.12130627  0.05802688]\n",
      " [ 0.16307104  0.23283623 -0.39590669]]\n",
      "1100 cost:0.794291, W:[[-0.9910726  -0.25094751  1.24201965]\n",
      " [ 0.06408136 -0.12084095  0.05676007]\n",
      " [ 0.16518959  0.23328991 -0.39847893]]\n",
      "1120 cost:0.791547, W:[[-1.00549901 -0.25430182  1.25980031]\n",
      " [ 0.06488273 -0.12035523  0.05547297]\n",
      " [ 0.16728874  0.23371412 -0.40100217]]\n",
      "1140 cost:0.788842, W:[[-1.01983476 -0.2576111   1.27744532]\n",
      " [ 0.0656836  -0.11985066  0.05416753]\n",
      " [ 0.16936915  0.23411041 -0.40347886]]\n",
      "1160 cost:0.786177, W:[[-1.0340811  -0.26087573  1.29495621]\n",
      " [ 0.06648365 -0.11932874  0.05284556]\n",
      " [ 0.1714315   0.23448041 -0.40591118]]\n",
      "1180 cost:0.783549, W:[[-1.04823959 -0.26409611  1.31233513]\n",
      " [ 0.06728257 -0.11879084  0.05150874]\n",
      " [ 0.1734764   0.23482558 -0.40830123]]\n",
      "1200 cost:0.780959, W:[[-1.06231129 -0.26727253  1.32958329]\n",
      " [ 0.06808005 -0.11823834  0.05015875]\n",
      " [ 0.17550454  0.23514733 -0.41065112]]\n",
      "1220 cost:0.778405, W:[[-1.07629776 -0.2704055   1.34670269]\n",
      " [ 0.06887576 -0.11767244  0.04879712]\n",
      " [ 0.17751651  0.23544692 -0.4129627 ]]\n",
      "1240 cost:0.775888, W:[[-1.09020054 -0.27349532  1.36369514]\n",
      " [ 0.06966945 -0.11709431  0.04742527]\n",
      " [ 0.17951292  0.23572563 -0.41523775]]\n",
      "1260 cost:0.773405, W:[[-1.10402048 -0.2765424   1.38056219]\n",
      " [ 0.07046084 -0.11650503  0.0460446 ]\n",
      " [ 0.18149434  0.23598464 -0.41747817]]\n",
      "1280 cost:0.770956, W:[[-1.11775899 -0.27954715  1.39730537]\n",
      " [ 0.07124964 -0.11590568  0.04465642]\n",
      " [ 0.18346132  0.23622501 -0.41968554]]\n",
      "1300 cost:0.768541, W:[[-1.13141763 -0.28250983  1.41392648]\n",
      " [ 0.0720356  -0.11529719  0.04326195]\n",
      " [ 0.18541445  0.23644781 -0.42186147]]\n",
      "1320 cost:0.766158, W:[[-1.144997   -0.28543106  1.43042743]\n",
      " [ 0.0728185  -0.11468047  0.04186235]\n",
      " [ 0.18735425  0.23665406 -0.42400748]]\n",
      "1340 cost:0.763808, W:[[-1.15849864 -0.28831109  1.44680917]\n",
      " [ 0.07359805 -0.11405637  0.04045873]\n",
      " [ 0.18928123  0.23684466 -0.42612502]]\n",
      "1360 cost:0.761489, W:[[-1.17192352 -0.29115036  1.46307349]\n",
      " [ 0.07437402 -0.11342576  0.03905213]\n",
      " [ 0.19119582  0.23702048 -0.42821544]]\n",
      "1380 cost:0.759201, W:[[-1.18527305 -0.29394919  1.47922182]\n",
      " [ 0.07514623 -0.11278931  0.03764351]\n",
      " [ 0.19309859  0.23718233 -0.43028003]]\n",
      "1400 cost:0.756943, W:[[-1.19854808 -0.29670808  1.49525583]\n",
      " [ 0.07591439 -0.11214777  0.03623381]\n",
      " [ 0.19498996  0.237331   -0.43232018]]\n",
      "1420 cost:0.754714, W:[[-1.21174991 -0.29942733  1.51117682]\n",
      " [ 0.07667837 -0.11150182  0.03482388]\n",
      " [ 0.19687036  0.2374672  -0.43433684]]\n",
      "1440 cost:0.752515, W:[[-1.22487962 -0.30210733  1.52698624]\n",
      " [ 0.07743796 -0.11085207  0.03341452]\n",
      " [ 0.19874024  0.23759168 -0.43633121]]\n",
      "1460 cost:0.750344, W:[[-1.23793781 -0.30474856  1.54268575]\n",
      " [ 0.07819301 -0.11019909  0.03200651]\n",
      " [ 0.20060001  0.2377051  -0.43830439]]\n",
      "1480 cost:0.7482, W:[[-1.25092602 -0.30735129  1.55827665]\n",
      " [ 0.07894332 -0.10954344  0.03060055]\n",
      " [ 0.20245005  0.23780803 -0.44025737]]\n",
      "1500 cost:0.746084, W:[[-1.26384497 -0.30991599  1.57376027]\n",
      " [ 0.07968874 -0.10888562  0.02919733]\n",
      " [ 0.20429076  0.23790109 -0.44219112]]\n",
      "1520 cost:0.743994, W:[[-1.27669585 -0.31244302  1.58913803]\n",
      " [ 0.08042911 -0.10822612  0.02779747]\n",
      " [ 0.2061225   0.23798482 -0.44410652]]\n",
      "1540 cost:0.741931, W:[[-1.28947926 -0.31493279  1.60441124]\n",
      " [ 0.0811643  -0.10756542  0.02640157]\n",
      " [ 0.2079456   0.23805973 -0.44600454]]\n",
      "1560 cost:0.739893, W:[[-1.30219638 -0.31738558  1.61958134]\n",
      " [ 0.08189417 -0.1069039   0.02501018]\n",
      " [ 0.2097604   0.23812631 -0.44788593]]\n",
      "1580 cost:0.737881, W:[[-1.31484818 -0.3198019   1.6346494 ]\n",
      " [ 0.0826186  -0.10624197  0.02362383]\n",
      " [ 0.21156724  0.238185   -0.44975147]]\n",
      "1600 cost:0.735892, W:[[-1.32743537 -0.32218221  1.64961684]\n",
      " [ 0.08333746 -0.10557999  0.022243  ]\n",
      " [ 0.21336642  0.23823628 -0.45160189]]\n",
      "1620 cost:0.733929, W:[[-1.33995903 -0.32452676  1.66448522]\n",
      " [ 0.08405066 -0.10491829  0.0208681 ]\n",
      " [ 0.21515822  0.2382805  -0.45343792]]\n",
      "1640 cost:0.731989, W:[[-1.35241997 -0.32683578  1.67925525]\n",
      " [ 0.08475811 -0.1042572   0.01949959]\n",
      " [ 0.21694297  0.23831807 -0.45526022]]\n",
      "1660 cost:0.730072, W:[[-1.36481905 -0.32910976  1.69392788]\n",
      " [ 0.08545969 -0.10359704  0.01813783]\n",
      " [ 0.21872087  0.23834935 -0.4570694 ]]\n",
      "1680 cost:0.728178, W:[[-1.37715721 -0.33134913  1.7085048 ]\n",
      " [ 0.08615532 -0.10293809  0.01678323]\n",
      " [ 0.22049227  0.23837472 -0.45886618]]\n",
      "1700 cost:0.726306, W:[[-1.38943517 -0.33355421  1.72298789]\n",
      " [ 0.08684494 -0.10228061  0.01543614]\n",
      " [ 0.22225733  0.23839442 -0.46065104]]\n",
      "1720 cost:0.724456, W:[[-1.40165365 -0.33572537  1.73737752]\n",
      " [ 0.08752849 -0.10162476  0.01409676]\n",
      " [ 0.22401641  0.23840885 -0.46242446]]\n",
      "1740 cost:0.722628, W:[[-1.41381371 -0.33786294  1.75167501]\n",
      " [ 0.08820589 -0.10097084  0.01276542]\n",
      " [ 0.22576959  0.23841822 -0.46418703]]\n",
      "1760 cost:0.720821, W:[[-1.42591584 -0.33996731  1.76588154]\n",
      " [ 0.08887712 -0.10031902  0.01144238]\n",
      " [ 0.22751717  0.23842283 -0.46593919]]\n",
      "1780 cost:0.719035, W:[[-1.43796086 -0.34203887  1.77999818]\n",
      " [ 0.08954211 -0.09966949  0.01012788]\n",
      " [ 0.22925931  0.23842292 -0.46768141]]\n",
      "1800 cost:0.717269, W:[[-1.44994974 -0.34407791  1.79402602]\n",
      " [ 0.09020081 -0.09902246  0.00882213]\n",
      " [ 0.23099625  0.23841871 -0.46941414]]\n",
      "1820 cost:0.715523, W:[[-1.46188295 -0.34608483  1.80796611]\n",
      " [ 0.09085319 -0.09837805  0.00752535]\n",
      " [ 0.23272817  0.23841047 -0.47113779]]\n",
      "1840 cost:0.713796, W:[[-1.47376132 -0.34805995  1.82181954]\n",
      " [ 0.09149921 -0.09773643  0.00623772]\n",
      " [ 0.23445523  0.23839836 -0.47285274]]\n",
      "1860 cost:0.712089, W:[[-1.48558569 -0.3500036   1.8355875 ]\n",
      " [ 0.09213884 -0.0970977   0.00495939]\n",
      " [ 0.23617762  0.23838258 -0.47455934]]\n",
      "1880 cost:0.710401, W:[[-1.49735653 -0.35191613  1.84927094]\n",
      " [ 0.09277208 -0.09646208  0.00369051]\n",
      " [ 0.2378955   0.23836336 -0.47625798]]\n",
      "1900 cost:0.708731, W:[[-1.50907469 -0.35379785  1.86287093]\n",
      " [ 0.09339891 -0.0958296   0.00243121]\n",
      " [ 0.23960902  0.23834084 -0.47794896]]\n",
      "1920 cost:0.70708, W:[[ -1.52074063e+00  -3.55649143e-01   1.87638831e+00]\n",
      " [  9.40192938e-02  -9.52003896e-02   1.18161971e-03]\n",
      " [  2.41318300e-01   2.38315195e-01  -4.79632616e-01]]\n",
      "1940 cost:0.705446, W:[[ -1.53235543e+00  -3.57470363e-01   1.88982415e+00]\n",
      " [  9.46332663e-02  -9.45745558e-02  -5.81700297e-05]\n",
      " [  2.43023530e-01   2.38286555e-01  -4.81309146e-01]]\n",
      "1960 cost:0.70383, W:[[ -1.54391921e+00  -3.59261781e-01   1.90317941e+00]\n",
      " [  9.52407792e-02  -9.39521641e-02  -1.28807756e-03]\n",
      " [  2.44724810e-01   2.38255069e-01  -4.82978940e-01]]\n",
      "1980 cost:0.702231, W:[[-1.55543303 -0.36102375  1.91645503]\n",
      " [ 0.09584185 -0.0933333  -0.00250801]\n",
      " [ 0.24642226  0.23822089 -0.48464221]]\n",
      "2000 cost:0.700649, W:[[-1.56689739 -0.36275655  1.92965221]\n",
      " [ 0.09643649 -0.09271803 -0.00371792]\n",
      " [ 0.24811605  0.23818412 -0.48629922]]\n",
      "a : [[ 0.68849677  0.26731515  0.04418808]], arg_max(a) : [0]\n",
      "b : [[ 0.24322268  0.44183081  0.3149465 ]], arg_max(c) : [1]\n",
      "c : [[ 0.02974809  0.08208466  0.8881672 ]], arg_max(b) : [2]\n",
      "all_data : [[ 0.68849677  0.26731515  0.04418808]\n",
      " [ 0.24322268  0.44183081  0.3149465 ]\n",
      " [ 0.02974809  0.08208466  0.8881672 ]], arg_max(all_data) : [0 1 2]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from IPython.display import display\n",
    "\n",
    "xy = np.loadtxt('data/lab06_train.txt', unpack=True, dtype='float32')\n",
    "x_data = np.transpose(xy[0:3])\n",
    "y_data = np.transpose(xy[3:])\n",
    "\n",
    "X = tf.placeholder('float', [None, 3])\n",
    "Y = tf.placeholder('float', [None, 3])\n",
    "\n",
    "W = tf.Variable(tf.zeros([3,3]))\n",
    "\n",
    "hypothesis = tf.nn.softmax(tf.matmul(X, W))\n",
    "learning_rate = 0.01\n",
    "cost = tf.reduce_mean(-tf.reduce_sum(Y * tf.log(hypothesis), reduction_indices=1))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for step in range(2001):\n",
    "        sess.run(optimizer, feed_dict={X:x_data, Y:y_data})\n",
    "        if step % 20 == 0:\n",
    "            print(step, \"cost:%s, W:%s\"%(sess.run(cost, feed_dict={X:x_data, Y:y_data}), sess.run(W)))\n",
    "    \n",
    "    a = sess.run(hypothesis, feed_dict={X:[[1,11,7]]})\n",
    "    print(\"a : %s, arg_max(a) : %s\" % (a, sess.run(tf.arg_max(a, 1))))\n",
    "    b = sess.run(hypothesis, feed_dict={X:[[1,3,4]]})\n",
    "    print(\"b : %s, arg_max(c) : %s\" % (b, sess.run(tf.arg_max(b, 1))))\n",
    "    c = sess.run(hypothesis, feed_dict={X:[[1,1,0]]})\n",
    "    print(\"c : %s, arg_max(b) : %s\" % (c, sess.run(tf.arg_max(c, 1))))\n",
    "    all_data = sess.run(hypothesis, feed_dict={X:[[1,11,7], [1,3,4], [1,1,0]]})\n",
    "    print(\"all_data : %s, arg_max(all_data) : %s\" % (all_data, sess.run(tf.arg_max(all_data, 1))))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning rate = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 cost:1.0678, W:[[-0.00833333  0.00416667  0.00416666]\n",
      " [ 0.01666667  0.02916667 -0.04583334]\n",
      " [ 0.01666666  0.04166667 -0.05833334]]\n",
      "20 cost:0.982314, W:[[-0.22666728 -0.05302757  0.27969486]\n",
      " [ 0.02956737 -0.06615412  0.03658675]\n",
      " [ 0.04377556  0.12877959 -0.17255519]]\n",
      "40 cost:0.923448, W:[[-0.42429063 -0.10565678  0.5299474 ]\n",
      " [ 0.0379938  -0.10515484  0.06716105]\n",
      " [ 0.07614058  0.18036503 -0.25650567]]\n",
      "60 cost:0.877164, W:[[-0.60482991 -0.15392429  0.75875419]\n",
      " [ 0.04507854 -0.12099476  0.07591628]\n",
      " [ 0.10603701  0.20820235 -0.31423941]]\n",
      "80 cost:0.839104, W:[[-0.77115828 -0.19723117  0.96838969]\n",
      " [ 0.05249171 -0.12503646  0.07254482]\n",
      " [ 0.13243006  0.22329152 -0.35572165]]\n",
      "100 cost:0.807015, W:[[-0.92562848 -0.2355711   1.16120005]\n",
      " [ 0.06036475 -0.12306247  0.06269784]\n",
      " [ 0.15575017  0.23137736 -0.38712767]]\n",
      "120 cost:0.779512, W:[[-1.07010567 -0.2692014   1.33930802]\n",
      " [ 0.06839757 -0.11813577  0.04973843]\n",
      " [ 0.17676352  0.23556805 -0.41233167]]\n",
      "140 cost:0.755648, W:[[-1.20607209 -0.29848364  1.50455654]\n",
      " [ 0.07625528 -0.11190803  0.03565298]\n",
      " [ 0.19616528  0.23758505 -0.43375036]]\n",
      "160 cost:0.734725, W:[[-1.33471942 -0.32380903  1.65852916]\n",
      " [ 0.08368905 -0.10526637  0.02157754]\n",
      " [ 0.21447927  0.23838788 -0.45286718]]\n",
      "180 cost:0.716209, W:[[-1.45701921 -0.34556246  1.80258262]\n",
      " [ 0.09055112 -0.09867419  0.00812325]\n",
      " [ 0.2320652   0.23850822 -0.4705734 ]]\n",
      "200 cost:0.699681, W:[[-1.5737735  -0.36410642  1.93788099]\n",
      " [ 0.0967759  -0.09235884 -0.00441682]\n",
      " [ 0.24915566  0.23823613 -0.48739177]]\n",
      "220 cost:0.68481, W:[[-1.68565142 -0.37977338  2.06542587]\n",
      " [ 0.10235459 -0.0864177  -0.01593657]\n",
      " [ 0.26589409  0.23772685 -0.50362092]]\n",
      "240 cost:0.67133, W:[[-1.79321432 -0.3928642   2.1860795 ]\n",
      " [ 0.10731332 -0.08087859 -0.02643434]\n",
      " [ 0.28236532  0.23706132 -0.51942664]]\n",
      "260 cost:0.659028, W:[[-1.89693725 -0.40364847  2.3005867 ]\n",
      " [ 0.11169729 -0.07573386 -0.03596294]\n",
      " [ 0.29861739  0.23628084 -0.5348981 ]]\n",
      "280 cost:0.647729, W:[[-1.99722362 -0.41236666  2.40959144]\n",
      " [ 0.11555995 -0.07095941 -0.0446    ]\n",
      " [ 0.31467584  0.23540626 -0.55008209]]\n",
      "300 cost:0.637292, W:[[-2.09441853 -0.41923243  2.51365304]\n",
      " [ 0.11895648 -0.06652464 -0.05243125]\n",
      " [ 0.33055362  0.23444836 -0.5650019 ]]\n",
      "320 cost:0.627601, W:[[-2.18881893 -0.4244357   2.61325669]\n",
      " [ 0.1219399  -0.06239806 -0.05954123]\n",
      " [ 0.34625652  0.23341316 -0.57966948]]\n",
      "340 cost:0.618557, W:[[-2.28067875 -0.42814526  2.70882654]\n",
      " [ 0.12455914 -0.05854956 -0.06600899]\n",
      " [ 0.36178681  0.23230478 -0.59409136]]\n",
      "360 cost:0.610083, W:[[-2.37022042 -0.43051097  2.80073357]\n",
      " [ 0.12685838 -0.05495156 -0.07190615]\n",
      " [ 0.37714529  0.23112696 -0.60827184]]\n",
      "380 cost:0.602109, W:[[-2.45763493 -0.43166646  2.88930321]\n",
      " [ 0.12887666 -0.05157946 -0.07729647]\n",
      " [ 0.3923322   0.2298831  -0.62221485]]\n",
      "400 cost:0.594579, W:[[-2.54309034 -0.43173078  2.97482228]\n",
      " [ 0.13064831 -0.04841127 -0.08223619]\n",
      " [ 0.40734777  0.22857714 -0.63592446]]\n",
      "420 cost:0.587445, W:[[-2.6267333  -0.43081033  3.05754447]\n",
      " [ 0.13220344 -0.04542786 -0.08677469]\n",
      " [ 0.42219296  0.22721291 -0.64940518]]\n",
      "440 cost:0.580666, W:[[-2.7086935  -0.42900032  3.13769484]\n",
      " [ 0.13356812 -0.04261214 -0.09095505]\n",
      " [ 0.43686876  0.22579454 -0.66266257]]\n",
      "460 cost:0.574207, W:[[-2.78908491 -0.426386    3.2154727 ]\n",
      " [ 0.13476501 -0.03994917 -0.09481487]\n",
      " [ 0.45137677  0.22432604 -0.67570227]]\n",
      "480 cost:0.568037, W:[[-2.86800981 -0.42304432  3.29105592]\n",
      " [ 0.13581407 -0.03742573 -0.09838732]\n",
      " [ 0.4657191   0.22281156 -0.68853009]]\n",
      "500 cost:0.562131, W:[[-2.94555783 -0.41904429  3.364604  ]\n",
      " [ 0.13673241 -0.03503014 -0.10170122]\n",
      " [ 0.47989804  0.22125506 -0.70115227]]\n",
      "520 cost:0.556464, W:[[-3.02181077 -0.41444826  3.4362607 ]\n",
      " [ 0.13753507 -0.032752   -0.10478208]\n",
      " [ 0.493916    0.21966036 -0.7135756 ]]\n",
      "540 cost:0.551017, W:[[-3.09684086 -0.40931255  3.50615501]\n",
      " [ 0.13823523 -0.03058199 -0.10765224]\n",
      " [ 0.5077759   0.21803123 -0.72580636]]\n",
      "560 cost:0.545772, W:[[-3.17071247 -0.40368828  3.57440281]\n",
      " [ 0.13884433 -0.02851182 -0.1103315 ]\n",
      " [ 0.52148032  0.21637124 -0.73785079]]\n",
      "580 cost:0.540714, W:[[-3.24348593 -0.39762163  3.64110899]\n",
      " [ 0.13937263 -0.02653413 -0.11283747]\n",
      " [ 0.53503233  0.21468362 -0.74971533]]\n",
      "600 cost:0.535829, W:[[-3.31521416 -0.39115471  3.70637012]\n",
      " [ 0.13982886 -0.02464214 -0.11518568]\n",
      " [ 0.54843497  0.21297167 -0.76140594]]\n",
      "620 cost:0.531104, W:[[-3.38594556 -0.38432562  3.77027249]\n",
      " [ 0.14022096 -0.02282986 -0.11739   ]\n",
      " [ 0.56169111  0.21123832 -0.77292871]]\n",
      "640 cost:0.526529, W:[[-3.45572472 -0.37716937  3.83289552]\n",
      " [ 0.14055575 -0.02109186 -0.11946284]\n",
      " [ 0.57480377  0.20948641 -0.7842893 ]]\n",
      "660 cost:0.522094, W:[[-3.52459264 -0.36971757  3.89431119]\n",
      " [ 0.14083955 -0.01942318 -0.12141529]\n",
      " [ 0.58777577  0.20771842 -0.79549336]]\n",
      "680 cost:0.517789, W:[[-3.59258604 -0.3619993   3.95458651]\n",
      " [ 0.1410778  -0.01781933 -0.12325735]\n",
      " [ 0.60061002  0.2059368  -0.80654609]]\n",
      "700 cost:0.513608, W:[[-3.65973949 -0.35404119  4.01378155]\n",
      " [ 0.1412753  -0.01627617 -0.12499794]\n",
      " [ 0.61330938  0.20414388 -0.81745255]]\n",
      "720 cost:0.509542, W:[[-3.72608471 -0.3458674   4.07195282]\n",
      " [ 0.1414364  -0.01478995 -0.12664521]\n",
      " [ 0.62587667  0.20234175 -0.82821769]]\n",
      "740 cost:0.505585, W:[[-3.79165053 -0.3375003   4.12915087]\n",
      " [ 0.14156488 -0.01335732 -0.12820631]\n",
      " [ 0.6383146   0.20053223 -0.83884609]]\n",
      "760 cost:0.501732, W:[[-3.85646486 -0.32896015  4.18542433]\n",
      " [ 0.14166433 -0.01197505 -0.12968801]\n",
      " [ 0.65062582  0.19871728 -0.84934217]]\n",
      "780 cost:0.497975, W:[[-3.92055225 -0.32026589  4.24081802]\n",
      " [ 0.14173782 -0.01064026 -0.13109623]\n",
      " [ 0.66281301  0.19689861 -0.85971057]]\n",
      "800 cost:0.494312, W:[[-3.98393679 -0.31143472  4.29537106]\n",
      " [ 0.1417881  -0.00935038 -0.13243635]\n",
      " [ 0.6748786   0.1950776  -0.86995488]]\n",
      "820 cost:0.490737, W:[[-4.04664135 -0.3024824   4.34912205]\n",
      " [ 0.14181767 -0.00810284 -0.13371348]\n",
      " [ 0.68682498  0.19325574 -0.88007945]]\n",
      "840 cost:0.487245, W:[[-4.10868359 -0.29342365  4.40210533]\n",
      " [ 0.14182867 -0.0068955  -0.13493183]\n",
      " [ 0.69865465  0.19143431 -0.89008754]]\n",
      "860 cost:0.483834, W:[[-4.17008495 -0.28427175  4.45435524]\n",
      " [ 0.14182322 -0.00572612 -0.13609584]\n",
      " [ 0.71036983  0.18961471 -0.89998305]]\n",
      "880 cost:0.480498, W:[[-4.23086405 -0.27503905  4.50590134]\n",
      " [ 0.14180322 -0.0045929  -0.13720904]\n",
      " [ 0.72197288  0.18779787 -0.90976924]]\n",
      "900 cost:0.477236, W:[[ -4.29103613e+00  -2.65737057e-01   4.55677176e+00]\n",
      " [  1.41770139e-01  -3.49386316e-03  -1.38274893e-01]\n",
      " [  7.33465910e-01   1.85984969e-01  -9.19449270e-01]]\n",
      "920 cost:0.474043, W:[[ -4.35061884e+00  -2.56376237e-01   4.60699368e+00]\n",
      " [  1.41725585e-01  -2.42750696e-03  -1.39296666e-01]\n",
      " [  7.44851112e-01   1.84176803e-01  -9.29026246e-01]]\n",
      "940 cost:0.470918, W:[[ -4.40962744e+00  -2.46966258e-01   4.65659285e+00]\n",
      " [  1.41670838e-01  -1.39211759e-03  -1.40277237e-01]\n",
      " [  7.56130576e-01   1.82374448e-01  -9.38503325e-01]]\n",
      "960 cost:0.467856, W:[[ -4.46807671e+00  -2.37516135e-01   4.70559168e+00]\n",
      " [  1.41607121e-01  -3.86307278e-04  -1.41219169e-01]\n",
      " [  7.67306447e-01   1.80578515e-01  -9.47883308e-01]]\n",
      "980 cost:0.464856, W:[[ -4.52598000e+00  -2.28034005e-01   4.75401449e+00]\n",
      " [  1.41535461e-01   5.91325108e-04  -1.42125010e-01]\n",
      " [  7.78380632e-01   1.78790003e-01  -9.57168996e-01]]\n",
      "1000 cost:0.461916, W:[[ -4.58334923e+00  -2.18527570e-01   4.80187750e+00]\n",
      " [  1.41456679e-01   1.54198625e-03  -1.42996773e-01]\n",
      " [  7.89354861e-01   1.77009344e-01  -9.66362596e-01]]\n",
      "1020 cost:0.459032, W:[[ -4.64019871e+00  -2.09003881e-01   4.84920216e+00]\n",
      " [  1.41371876e-01   2.46676104e-03  -1.43836826e-01]\n",
      " [  8.00231159e-01   1.75237149e-01  -9.75466788e-01]]\n",
      "1040 cost:0.456204, W:[[ -4.69653940e+00  -1.99469447e-01   4.89600897e+00]\n",
      " [  1.41281873e-01   3.36697605e-03  -1.44647047e-01]\n",
      " [  8.11011434e-01   1.73474148e-01  -9.84484136e-01]]\n",
      "1060 cost:0.453429, W:[[ -4.75238276e+00  -1.89930201e-01   4.94231272e+00]\n",
      " [  1.41187236e-01   4.24363371e-03  -1.45429000e-01]\n",
      " [  8.21697474e-01   1.71720937e-01  -9.93416667e-01]]\n",
      "1080 cost:0.450705, W:[[-4.80773973 -0.18039171  4.988132  ]\n",
      " [ 0.14108872  0.00509769 -0.14618447]\n",
      " [ 0.83229083  0.16997786 -1.00226688]]\n",
      "1100 cost:0.44803, W:[[-4.86261988 -0.17085913  5.03348017]\n",
      " [ 0.14098684  0.00592999 -0.14691487]\n",
      " [ 0.84279329  0.16824533 -1.01103675]]\n",
      "1120 cost:0.445403, W:[[-4.91703367 -0.16133717  5.078372  ]\n",
      " [ 0.14088213  0.00674146 -0.14762163]\n",
      " [ 0.85320628  0.16652377 -1.01972818]]\n",
      "1140 cost:0.442822, W:[[-4.97099113 -0.15183011  5.12282276]\n",
      " [ 0.14077517  0.00753289 -0.14830607]\n",
      " [ 0.86353171  0.16481353 -1.02834332]]\n",
      "1160 cost:0.440286, W:[[-5.02450132 -0.14234203  5.16684484]\n",
      " [ 0.14066635  0.00830512 -0.14896944]\n",
      " [ 0.87377095  0.16311499 -1.03688395]]\n",
      "1180 cost:0.437793, W:[[-5.07757282 -0.13287663  5.21045113]\n",
      " [ 0.14055608  0.00905886 -0.14961287]\n",
      " [ 0.88392556  0.16142838 -1.04535174]]\n",
      "1200 cost:0.435343, W:[[-5.13021469 -0.12343737  5.25365353]\n",
      " [ 0.14044461  0.00979479 -0.15023731]\n",
      " [ 0.89399701  0.15975396 -1.05374885]]\n",
      "1220 cost:0.432933, W:[[-5.18243504 -0.11402738  5.29646397]\n",
      " [ 0.14033233  0.01051354 -0.15084375]\n",
      " [ 0.90398687  0.15809196 -1.06207669]]\n",
      "1240 cost:0.430562, W:[[-5.23424101 -0.1046495   5.33889294]\n",
      " [ 0.14021966  0.01121579 -0.15143324]\n",
      " [ 0.91389638  0.15644267 -1.07033694]]\n",
      "1260 cost:0.42823, W:[[-5.28564119 -0.09530647  5.38094997]\n",
      " [ 0.14010657  0.011902   -0.15200639]\n",
      " [ 0.92372698  0.15480618 -1.07853103]]\n",
      "1280 cost:0.425935, W:[[-5.33664227 -0.0860007   5.42264509]\n",
      " [ 0.13999353  0.01257285 -0.15256418]\n",
      " [ 0.93348014  0.1531827  -1.0866605 ]]\n",
      "1300 cost:0.423676, W:[[-5.38725376 -0.07673445  5.46398926]\n",
      " [ 0.13988075  0.01322881 -0.1531073 ]\n",
      " [ 0.94315708  0.15157236 -1.09472692]]\n",
      "1320 cost:0.421453, W:[[-5.43747616 -0.06750979  5.50498962]\n",
      " [ 0.13976806  0.01387041 -0.15363626]\n",
      " [ 0.95275885  0.14997536 -1.1027317 ]]\n",
      "1340 cost:0.419263, W:[[-5.48732471 -0.05832863  5.54565525]\n",
      " [ 0.13965613  0.014498   -0.15415184]\n",
      " [ 0.96228701  0.14839154 -1.11067629]]\n",
      "1360 cost:0.417107, W:[[-5.53680038 -0.04919266  5.5859952 ]\n",
      " [ 0.13954473  0.01511229 -0.15465467]\n",
      " [ 0.97174275  0.14682133 -1.11856163]]\n",
      "1380 cost:0.414983, W:[[-5.5859108  -0.04010348  5.62601662]\n",
      " [ 0.13943416  0.01571354 -0.15514526]\n",
      " [ 0.98112714  0.14526463 -1.12638927]]\n",
      "1400 cost:0.412891, W:[[-5.63466215 -0.03106253  5.66572714]\n",
      " [ 0.1393245   0.01630212 -0.15562418]\n",
      " [ 0.99044132  0.14372151 -1.13416028]]\n",
      "1420 cost:0.410829, W:[[-5.68306017 -0.02207111  5.70513391]\n",
      " [ 0.13921589  0.0168785  -0.15609188]\n",
      " [ 0.99968648  0.14219204 -1.14187586]]\n",
      "1440 cost:0.408797, W:[[-5.73111105 -0.01313043  5.74424458]\n",
      " [ 0.13910845  0.01744303 -0.15654895]\n",
      " [ 1.00886369  0.14067625 -1.14953732]]\n",
      "1460 cost:0.406795, W:[[ -5.77882051e+00  -4.24156431e-03   5.78306389e+00]\n",
      " [  1.39002129e-01   1.79960523e-02  -1.56995609e-01]\n",
      " [  1.01797402e+00   1.39174029e-01  -1.15714550e+00]]\n",
      "1480 cost:0.40482, W:[[ -5.82619429e+00   4.59453324e-03   5.82160091e+00]\n",
      " [  1.38897076e-01   1.85380261e-02  -1.57432452e-01]\n",
      " [  1.02701890e+00   1.37685597e-01  -1.16470158e+00]]\n",
      "1500 cost:0.402874, W:[[-5.87323666  0.01337698  5.85986137]\n",
      " [ 0.13879329  0.01906913 -0.15785977]\n",
      " [ 1.03599894  0.13621086 -1.17220688]]\n",
      "1520 cost:0.400955, W:[[-5.91995382  0.022105    5.89785051]\n",
      " [ 0.13869095  0.01958977 -0.15827794]\n",
      " [ 1.0449152   0.13474983 -1.17966211]]\n",
      "1540 cost:0.399062, W:[[-5.96635103  0.03077788  5.93557453]\n",
      " [ 0.13859008  0.02010013 -0.1586874 ]\n",
      " [ 1.05376887  0.13330244 -1.18706858]]\n",
      "1560 cost:0.397194, W:[[-6.0124321   0.03939497  5.9730382 ]\n",
      " [ 0.13849038  0.02060053 -0.15908802]\n",
      " [ 1.0625608   0.13186856 -1.19442725]]\n",
      "1580 cost:0.395352, W:[[-6.05820227  0.0479558   6.01024723]\n",
      " [ 0.13839233  0.02109135 -0.15948072]\n",
      " [ 1.0712918   0.13044831 -1.20173836]]\n",
      "1600 cost:0.393535, W:[[-6.10366583  0.05645988  6.04720736]\n",
      " [ 0.13829574  0.02157287 -0.15986556]\n",
      " [ 1.07996321  0.12904176 -1.20900333]]\n",
      "1620 cost:0.391741, W:[[-6.14882851  0.06490678  6.08392286]\n",
      " [ 0.13820067  0.02204522 -0.16024281]\n",
      " [ 1.08857548  0.12764874 -1.21622276]]\n",
      "1640 cost:0.389971, W:[[-6.19369411  0.07329617  6.12039852]\n",
      " [ 0.13810718  0.02250868 -0.16061281]\n",
      " [ 1.0971297   0.12626922 -1.22339761]]\n",
      "1660 cost:0.388224, W:[[-6.23826647  0.08162774  6.15664005]\n",
      " [ 0.13801517  0.0229635  -0.16097566]\n",
      " [ 1.10562694  0.12490318 -1.23052859]]\n",
      "1680 cost:0.386499, W:[[-6.28255033  0.08990127  6.19265079]\n",
      " [ 0.13792475  0.02340985 -0.16133164]\n",
      " [ 1.11406779  0.12355052 -1.23761678]]\n",
      "1700 cost:0.384796, W:[[-6.32655048  0.09811655  6.22843552]\n",
      " [ 0.13783596  0.02384803 -0.16168101]\n",
      " [ 1.12245297  0.12221122 -1.24466276]]\n",
      "1720 cost:0.383115, W:[[-6.37027025  0.10627342  6.26399851]\n",
      " [ 0.13774873  0.02427825 -0.1620239 ]\n",
      " [ 1.13078368  0.12088512 -1.25166738]]\n",
      "1740 cost:0.381455, W:[[-6.41371393  0.11437177  6.29934454]\n",
      " [ 0.13766298  0.0247007  -0.1623605 ]\n",
      " [ 1.1390605   0.11957222 -1.25863123]]\n",
      "1760 cost:0.379815, W:[[-6.45688534  0.12241157  6.33447599]\n",
      " [ 0.1375788   0.02511552 -0.16269115]\n",
      " [ 1.14728427  0.11827242 -1.26555514]]\n",
      "1780 cost:0.378195, W:[[-6.49978828  0.13039275  6.36939812]\n",
      " [ 0.13749631  0.02552291 -0.16301605]\n",
      " [ 1.15545547  0.11698564 -1.27243972]]\n",
      "1800 cost:0.376595, W:[[-6.54242659  0.13831529  6.40411282]\n",
      " [ 0.13741526  0.02592302 -0.16333508]\n",
      " [ 1.16357517  0.11571171 -1.27928567]]\n",
      "1820 cost:0.375014, W:[[-6.58480358  0.14617927  6.43862581]\n",
      " [ 0.13733575  0.02631611 -0.16364859]\n",
      " [ 1.17164385  0.11445064 -1.28609371]]\n",
      "1840 cost:0.373452, W:[[-6.62692356  0.15398477  6.47293997]\n",
      " [ 0.13725783  0.0267024  -0.16395684]\n",
      " [ 1.17966282  0.11320242 -1.29286432]]\n",
      "1860 cost:0.371909, W:[[-6.66878986  0.16173182  6.50705767]\n",
      " [ 0.1371813   0.02708187 -0.16425972]\n",
      " [ 1.1876322   0.11196678 -1.29959822]]\n",
      "1880 cost:0.370383, W:[[-6.71040487  0.16942063  6.54098463]\n",
      " [ 0.13710621  0.02745488 -0.16455762]\n",
      " [ 1.19555318  0.11074394 -1.30629599]]\n",
      "1900 cost:0.368876, W:[[-6.75177193  0.17705128  6.57472086]\n",
      " [ 0.13703267  0.0278214  -0.16485053]\n",
      " [ 1.20342577  0.10953357 -1.31295812]]\n",
      "1920 cost:0.367385, W:[[-6.79289532  0.18462394  6.60827255]\n",
      " [ 0.13696054  0.0281817  -0.16513868]\n",
      " [ 1.21125102  0.10833559 -1.31958544]]\n",
      "1940 cost:0.365912, W:[[-6.83377838  0.19213878  6.64164066]\n",
      " [ 0.1368899   0.02853582 -0.16542208]\n",
      " [ 1.21902966  0.10714997 -1.32617831]]\n",
      "1960 cost:0.364456, W:[[-6.87442398  0.19959597  6.67482853]\n",
      " [ 0.13682073  0.02888399 -0.16570097]\n",
      " [ 1.22676206  0.1059766  -1.33273733]]\n",
      "1980 cost:0.363015, W:[[-6.91483498  0.20699579  6.70783949]\n",
      " [ 0.13675293  0.02922632 -0.16597532]\n",
      " [ 1.23444915  0.10481534 -1.33926272]]\n",
      "2000 cost:0.361591, W:[[-6.95501471  0.21433842  6.7406764 ]\n",
      " [ 0.13668649  0.02956286 -0.16624542]\n",
      " [ 1.2420913   0.1036661  -1.3457557 ]]\n",
      "a : [[  8.78134191e-01   1.21488057e-01   3.77686898e-04]], arg_max(a) : [0]\n",
      "b : [[ 0.04476492  0.44394752  0.51128757]], arg_max(c) : [2]\n",
      "c : [[  1.52343887e-06   1.77792064e-03   9.98220503e-01]], arg_max(b) : [2]\n",
      "all_data : [[  8.78134191e-01   1.21488057e-01   3.77686898e-04]\n",
      " [  4.47649248e-02   4.43947524e-01   5.11287570e-01]\n",
      " [  1.52343887e-06   1.77792076e-03   9.98220503e-01]], arg_max(all_data) : [0 2 2]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from IPython.display import display\n",
    "\n",
    "xy = np.loadtxt('data/lab06_train.txt', unpack=True, dtype='float32')\n",
    "x_data = np.transpose(xy[0:3])\n",
    "y_data = np.transpose(xy[3:])\n",
    "\n",
    "X = tf.placeholder('float', [None, 3])\n",
    "Y = tf.placeholder('float', [None, 3])\n",
    "\n",
    "W = tf.Variable(tf.zeros([3,3]))\n",
    "\n",
    "hypothesis = tf.nn.softmax(tf.matmul(X, W))\n",
    "learning_rate = 0.1\n",
    "cost = tf.reduce_mean(-tf.reduce_sum(Y * tf.log(hypothesis), reduction_indices=1))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for step in range(2001):\n",
    "        sess.run(optimizer, feed_dict={X:x_data, Y:y_data})\n",
    "        if step % 20 == 0:\n",
    "            print(step, \"cost:%s, W:%s\"%(sess.run(cost, feed_dict={X:x_data, Y:y_data}), sess.run(W)))\n",
    "    \n",
    "    a = sess.run(hypothesis, feed_dict={X:[[1,11,7]]})\n",
    "    print(\"a : %s, arg_max(a) : %s\" % (a, sess.run(tf.arg_max(a, 1))))\n",
    "    b = sess.run(hypothesis, feed_dict={X:[[1,3,4]]})\n",
    "    print(\"b : %s, arg_max(c) : %s\" % (b, sess.run(tf.arg_max(b, 1))))\n",
    "    c = sess.run(hypothesis, feed_dict={X:[[1,1,0]]})\n",
    "    print(\"c : %s, arg_max(b) : %s\" % (c, sess.run(tf.arg_max(c, 1))))\n",
    "    all_data = sess.run(hypothesis, feed_dict={X:[[1,11,7], [1,3,4], [1,1,0]]})\n",
    "    print(\"all_data : %s, arg_max(all_data) : %s\" % (all_data, sess.run(tf.arg_max(all_data, 1))))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load MNIST datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting ./temp/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting ./temp/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting ./temp/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting ./temp/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets\n",
    "mnist = read_data_sets('./temp', one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 cost=0.529839764\n",
      "Epoch: 0002 cost=0.349446594\n",
      "Epoch: 0003 cost=0.323961806\n",
      "Epoch: 0004 cost=0.310680645\n",
      "Epoch: 0005 cost=0.297954461\n",
      "Epoch: 0006 cost=0.295289272\n",
      "Epoch: 0007 cost=0.287389971\n",
      "Epoch: 0008 cost=0.283680185\n",
      "Epoch: 0009 cost=0.279713999\n",
      "Epoch: 0010 cost=0.275662967\n",
      "Optimization Finished!\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder('float', [None, 784])\n",
    "y = tf.placeholder('float', [None, 10])\n",
    "\n",
    "W = tf.Variable(tf.zeros([784, 10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "\n",
    "activation = tf.nn.softmax(tf.matmul(x, W) + b)\n",
    "\n",
    "learning_rate=0.01\n",
    "cost = tf.reduce_mean(-tf.reduce_sum(y * tf.log(activation), reduction_indices=1))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "training_epochs = 10\n",
    "batch_size = 10\n",
    "display_step = 1\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "\n",
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0.\n",
    "    total_batch = int(mnist.train.num_examples / batch_size)\n",
    "    for i in range(total_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size=batch_size)\n",
    "        sess.run(optimizer, feed_dict={x : batch_xs, y: batch_ys})\n",
    "        avg_cost += sess.run(cost, feed_dict={x : batch_xs, y: batch_ys}) / total_batch\n",
    "\n",
    "    if epoch % display_step == 0:\n",
    "        print('Epoch: %04d' % (epoch+1), 'cost={:.9f}'.format(avg_cost))\n",
    "\n",
    "print('Optimization Finished!')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Predict & Show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label: [5]\n",
      "Prediction: [5]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADfFJREFUeJzt3W+sVPWdx/HPF6SGCKhXpoQ/sreSmyZKsnQz4p8S07VL\ntaaK9YGWkApR9zYRTat9sMbVLPGRWW2bxqxEupLSTde2hio8IBXETUwTaRiNK1p1/ZOLgFe4iIqY\nGFS+++Aemqve8zvDzJk5c/m+X8nkzpzvOXO+GfjMmTnnzPmZuwtAPJOqbgBANQg/EBThB4Ii/EBQ\nhB8IivADQRF+ICjCDwRF+IGgTunmymbOnOn9/f3dXCUQytDQkA4ePGjNzNtW+M3sckm/lDRZ0n+6\n+72p+fv7+9VoNNpZJYCEer3e9Lwtf+w3s8mS/kPSdyWdK2m5mZ3b6vMB6K52vvMvlvS6u7/p7kcl\n/U7SsnLaAtBp7YR/rqQ9Yx7vzaZ9jpkNmlnDzBojIyNtrA5AmTq+t9/d17l73d3rtVqt06sD0KR2\nwr9P0tljHs/LpgGYANoJ/05JA2b2NTP7iqQfSNpcTlsAOq3lQ33u/qmZ3SLpCY0e6lvv7i+V1hmA\njmrrOL+7b5G0paReAHQRp/cCQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4\ngaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF\n+IGgCD8QVFuj9JrZkKQPJX0m6VN3r5fRFIDOayv8mX9094MlPA+ALuJjPxBUu+F3SVvN7FkzGyyj\nIQDd0e7H/iXuvs/Mvippm5m94u5Pj50he1MYlKT58+e3uToAZWlry+/u+7K/ByQ9JmnxOPOsc/e6\nu9drtVo7qwNQopbDb2anmdn04/clfUfSi2U1BqCz2vnYP0vSY2Z2/Hn+293/VEpXADqu5fC7+5uS\n/r7EXgB0EYf6gKAIPxAU4QeCIvxAUIQfCIrwA0GV8as+nMTefffdZP2ss87q2LoPHz6crI+MjHRs\n3UWmTp2arM+ZM6dLnbSOLT8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBMVx/uDuu+++ZH3NmjXJ+qpV\nq5L1adOm5dZ27NiRXHbv3r3J+tDQULKe4u7JenadilynnJKOTl9fX7I+PDycrHcDW34gKMIPBEX4\ngaAIPxAU4QeCIvxAUIQfCIrj/BPAsWPHkvVXXnklt1Z0nH7btm3J+sUXX5ysr127NllPafdYezuK\n1j158uRkPXX+glT8b9YL2PIDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCFx/nNbL2k70k64O4Ls2l9\nkn4vqV/SkKRr3f29zrV5cnv77beT9dtvvz1Zf/TRR3NrV155ZXLZN954I1l///33k/WBgYFkfcaM\nGbm1I0eOJJddsWJFsn7ppZcm60uWLMmtFV3zf+7cucl6rVZL1ot+798Lmtny/1rS5V+Ydoek7e4+\nIGl79hjABFIYfnd/WtKhL0xeJmlDdn+DpKtL7gtAh7X6nX+Wux+/DtE7kmaV1A+ALml7h5+PniSd\ne6K0mQ2aWcPMGlWOrQbg81oN/34zmy1J2d8DeTO6+zp3r7t7vWgnCYDuaTX8myWtzO6vlLSpnHYA\ndEth+M3sEUnPSPq6me01sxsl3StpqZm9JumfsscAJpDCg5Huvjyn9O2SezlpFf3mffXq1cn6okWL\nkvUtW7bk1pYuXZpcdtKk9Pt/0e/a77///mT9pptuyq198sknyWWLrn3fjnPOOadjzz1RcIYfEBTh\nB4Ii/EBQhB8IivADQRF+IKje/93hSeCee+5pa/mpU6cm60899VRu7dVXX00uW3Rp7vPOOy9Zv+22\n25J19C62/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFMf5u+Caa65J1h988MFk/ZlnnmmrnlI0VPXp\np5+erG/alL6OyyWXXJJb6+QQ3CjGlh8IivADQRF+ICjCDwRF+IGgCD8QFOEHgrKi47xlqtfr3mg0\nura+iaJomOyi4+EffPBBbu2jjz5KLlt0HP+6665L1ouuF/DAAw/k1m6++ebksjhx9XpdjUajqRMo\n2PIDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCFv+c3s/WSvifpgLsvzKatkfTPkkay2e509/xxopG0\nYMGCqlvItWvXrmS9aPjxW2+9Nbd21VVXJZedN29eso72NLPl/7Wky8eZ/gt3X5TdCD4wwRSG392f\nlnSoC70A6KJ2vvPfYmYvmNl6MzuztI4AdEWr4V8raYGkRZKGJf0sb0YzGzSzhpk1RkZG8mYD0GUt\nhd/d97v7Z+5+TNKvJC1OzLvO3evuXq/Vaq32CaBkLYXfzGaPefh9SS+W0w6AbmnmUN8jkr4laaaZ\n7ZX0b5K+ZWaLJLmkIUk/6mCPADqgMPzuvnycyQ93oJdKbdy4MVmfM2dObu2iiy4qu52eMXny5GT9\n+uuvT9affPLJ3NrWrVuTy95www3JOtrDGX5AUIQfCIrwA0ERfiAowg8ERfiBoBiiO/PQQw8l66lL\nWF9wwQXJZSdNOnnfYz/++ONk/YknnsitXXbZZWW3gxNw8v6vBJBE+IGgCD8QFOEHgiL8QFCEHwiK\n8ANBcZw/c8YZZyTrg4ODubX33nsvueyKFSuS9b6+vmT91FNPTdZTjhw5kqxPmTIlWS86jr9q1apk\nfWBgoOVl0Vls+YGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKHP3rq2sXq97o9Ho2vpOxM6dO5P1Cy+8\nsGPrnj9/frJedA5Cyp49e5L16dOnJ+tHjx5N1mfMmJGsb9iwIbe2eHHuQE9oUb1eV6PRsGbmZcsP\nBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0EV/p7fzM6W9BtJsyS5pHXu/ksz65P0e0n9koYkXevu6R+2\n97B6vZ6s79ixI7fW7jkAu3fvTtbfeuuttp4/5dChQ8n6woULk/W77747WedYfu9qZsv/qaSfuvu5\nki6UtNrMzpV0h6Tt7j4gaXv2GMAEURh+dx929+ey+x9KelnSXEnLJB0/fWuDpKs71SSA8p3Qd34z\n65f0DUl/kTTL3Yez0jsa/VoAYIJoOvxmNk3SRkk/cffDY2s++gOBcX8kYGaDZtYws8bIyEhbzQIo\nT1PhN7MpGg3+b939j9nk/WY2O6vPlnRgvGXdfZ271929XqvVyugZQAkKw29mJulhSS+7+8/HlDZL\nWpndXylpU/ntAeiUZi7d/U1JP5S0y8yez6bdKeleSX8wsxsl7ZZ0bWda7I7R97h8559/fm6t6PLY\n+/fvT9aHh4eT9ccff7zl+uHDh3NrkrR69epk/a677krWMXEVht/d/ywpLxnfLrcdAN3CGX5AUIQf\nCIrwA0ERfiAowg8ERfiBoLh0N3AS4dLdAAoRfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+\nICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIXhN7Ozzex/\nzOyvZvaSmf04m77GzPaZ2fPZ7YrOtwugLKc0Mc+nkn7q7s+Z2XRJz5rZtqz2C3e/v3PtAeiUwvC7\n+7Ck4ez+h2b2sqS5nW4MQGed0Hd+M+uX9A1Jf8km3WJmL5jZejM7M2eZQTNrmFljZGSkrWYBlKfp\n8JvZNEkbJf3E3Q9LWitpgaRFGv1k8LPxlnP3de5ed/d6rVYroWUAZWgq/GY2RaPB/627/1GS3H2/\nu3/m7sck/UrS4s61CaBszeztN0kPS3rZ3X8+ZvrsMbN9X9KL5bcHoFOa2dv/TUk/lLTLzJ7Ppt0p\nabmZLZLkkoYk/agjHQLoiGb29v9Z0njjfW8pvx0A3cIZfkBQhB8IivADQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaDM3bu3MrMRSbvHTJop6WDXGjgxvdpbr/Yl0Vuryuzt\n79y9qevldTX8X1q5WcPd65U1kNCrvfVqXxK9taqq3vjYDwRF+IGgqg7/uorXn9KrvfVqXxK9taqS\n3ir9zg+gOlVv+QFUpJLwm9nlZvaqmb1uZndU0UMeMxsys13ZyMONintZb2YHzOzFMdP6zGybmb2W\n/R13mLSKeuuJkZsTI0tX+tr12ojXXf/Yb2aTJf2fpKWS9kraKWm5u/+1q43kMLMhSXV3r/yYsJld\nIumIpN+4+8Js2r9LOuTu92ZvnGe6+7/0SG9rJB2peuTmbECZ2WNHlpZ0taRVqvC1S/R1rSp43arY\n8i+W9Lq7v+nuRyX9TtKyCvroee7+tKRDX5i8TNKG7P4Gjf7n6bqc3nqCuw+7+3PZ/Q8lHR9ZutLX\nLtFXJaoI/1xJe8Y83qveGvLbJW01s2fNbLDqZsYxKxs2XZLekTSrymbGUThyczd9YWTpnnntWhnx\numzs8PuyJe7+D5K+K2l19vG2J/nod7ZeOlzT1MjN3TLOyNJ/U+Vr1+qI12WrIvz7JJ095vG8bFpP\ncPd92d8Dkh5T740+vP/4IKnZ3wMV9/M3vTRy83gjS6sHXrteGvG6ivDvlDRgZl8zs69I+oGkzRX0\n8SVmdlq2I0Zmdpqk76j3Rh/eLGlldn+lpE0V9vI5vTJyc97I0qr4teu5Ea/dves3SVdodI//G5L+\ntYoecvo6R9L/ZreXqu5N0iMa/Rj4iUb3jdwo6SxJ2yW9JulJSX091Nt/Sdol6QWNBm12Rb0t0ehH\n+hckPZ/drqj6tUv0Vcnrxhl+QFDs8AOCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/ENT/AwtzU4B3\nT3gUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113d5bf60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from random import randint\n",
    "\n",
    "r = randint(0, mnist.test.num_examples - 1)\n",
    "print('Label: %s' % sess.run(tf.argmax(mnist.test.labels[r:r+1], 1)))\n",
    "print('Prediction: %s' % sess.run(tf.argmax(activation, 1), {x:mnist.test.images[r:r+1]}))\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.imshow(mnist.test.images[r:r+1].reshape(28, 28), cmap='Greys', interpolation='nearest')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.921\n"
     ]
    }
   ],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(activation, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))\n",
    "print('Accuracy : %s' % accuracy.eval(session=sess, feed_dict={x:mnist.test.images, y:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}