lab-07-1-learning_rate_and_evaluation-tf2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "When lr: 1.5\n",
      "Step:  0 cost:  12.9678288 W:  [[1.14470661 0.93543148 -0.991899252]\n",
      " [2.43914 2.03430772 -3.48942089]\n",
      " [2.68254757 2.71287251 -3.80668187]]\n",
      "Step:  20 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  40 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  60 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  80 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  100 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  120 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  140 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  160 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  180 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Step:  200 cost:  nan W:  [[-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]\n",
      " [-nan(ind) -nan(ind) -nan(ind)]]\n",
      "Prediction: [0 0 0]\n",
      "Accuracy:  0\n",
      "-------------------------------------------------\n",
      "When lr: 1e-10\n",
      "Step:  0 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  20 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  40 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  60 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  80 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  100 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  120 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  140 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  160 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  180 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Step:  200 cost:  8.06258106 W:  [[0.352659762 -0.255153239 -1.13683784]\n",
      " [-0.747629523 -0.964119196 -0.267552048]\n",
      " [1.23969269 -1.07209778 -0.965970039]]\n",
      "Prediction: [0 0 0]\n",
      "Accuracy:  0\n",
      "-------------------------------------------------\n",
      "When lr: 0.1\n",
      "Step:  0 cost:  12.4496698 W:  [[-0.251980752 -2.27875829 1.42364025]\n",
      " [-0.968747854 0.0643920526 0.804940224]\n",
      " [-0.637479544 -2.02579808 0.694372177]]\n",
      "Step:  20 cost:  1.06507397 W:  [[-0.383268714 -2.12475181 1.40092218]\n",
      " [-0.289882094 0.455029279 -0.264562607]\n",
      " [-0.323257715 -0.959573328 -0.686074555]]\n",
      "Step:  40 cost:  0.867601395 W:  [[-0.587053835 -2.10434341 1.58429897]\n",
      " [-0.104571439 0.181556076 -0.176399976]\n",
      " [-0.427983373 -0.673751056 -0.867171]]\n",
      "Step:  60 cost:  0.775168419 W:  [[-0.755940497 -2.09018922 1.73903179]\n",
      " [-0.00205339422 0.0273772571 -0.12473911]\n",
      " [-0.463284492 -0.515981793 -0.989639103]]\n",
      "Step:  80 cost:  0.726179481 W:  [[-0.89828223 -2.08267665 1.87386072]\n",
      " [0.0496388 -0.0454960577 -0.103557855]\n",
      " [-0.458409697 -0.440475821 -1.07001984]]\n",
      "Step:  100 cost:  0.693039179 W:  [[-1.0227772 -2.07792592 1.99360466]\n",
      " [0.0766427517 -0.076585792 -0.099472113]\n",
      " [-0.436053723 -0.406471342 -1.12638044]]\n",
      "Step:  120 cost:  0.667165577 W:  [[-1.13500214 -2.07362461 2.10152888]\n",
      " [0.0919274092 -0.0878109783 -0.103531569]\n",
      " [-0.406953424 -0.392467439 -1.1694845]]\n",
      "Step:  140 cost:  0.645595074 W:  [[-1.23831189 -2.06876922 2.1999836]\n",
      " [0.101375513 -0.0898023322 -0.110988259]\n",
      " [-0.375654131 -0.38814792 -1.20510316]]\n",
      "Step:  160 cost:  0.626998 W:  [[-1.33479941 -2.06301022 2.29071236]\n",
      " [0.107698359 -0.0876349732 -0.119478442]\n",
      " [-0.344034046 -0.38855657 -1.2363143]]\n",
      "Step:  180 cost:  0.610622883 W:  [[-1.42584503 -2.05629253 2.37504029]\n",
      " [0.112194195 -0.0837295055 -0.127879724]\n",
      " [-0.312841177 -0.391283423 -1.26478052]]\n",
      "Step:  200 cost:  0.595980167 W:  [[-1.51240671 -2.04868698 2.45399642]\n",
      " [0.115518339 -0.0792304724 -0.135702819]\n",
      " [-0.282344759 -0.395144045 -1.29141581]]\n",
      "Prediction: [2 2 2]\n",
      "Accuracy:  1\n",
      "-------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\nwhen lr = 1.5\\n\\n0 5.73203 [[-0.30548954  1.22985029 -0.66033536]\\n [-4.39069986  2.29670858  2.99386835]\\n [-3.34510708  2.09743214 -0.80419564]]\\n1 23.1494 [[ 0.06951046  0.29449689 -0.0999819 ]\\n [-1.95319986 -1.63627958  4.48935604]\\n [-0.90760708 -1.65020132  0.50593793]]\\n2 27.2798 [[ 0.44451016  0.85699677 -1.03748143]\\n [ 0.48429942  0.98872018 -0.57314301]\\n [ 1.52989244  1.16229868 -4.74406147]]\\n3 8.668 [[ 0.12396193  0.61504567 -0.47498202]\\n [ 0.22003263 -0.2470119   0.9268558 ]\\n [ 0.96035379  0.41933775 -3.43156195]]\\n4 5.77111 [[-0.9524312   1.13037777  0.08607888]\\n [-3.78651619  2.26245379  2.42393875]\\n [-3.07170963  3.14037919 -2.12054014]]\\n5 inf [[ nan  nan  nan]\\n [ nan  nan  nan]\\n [ nan  nan  nan]]\\n6 nan [[ nan  nan  nan]\\n [ nan  nan  nan]\\n [ nan  nan  nan]]\\n ...\\nPrediction: [0 0 0]\\nAccuracy:  0.0\\n-------------------------------------------------\\nWhen lr = 1e-10\\n\\n0 5.73203 [[ 0.80269563  0.67861295 -1.21728313]\\n [-0.3051686  -0.3032113   1.50825703]\\n [ 0.75722361 -0.7008909  -2.10820389]]\\n1 5.73203 [[ 0.80269563  0.67861295 -1.21728313]\\n [-0.3051686  -0.3032113   1.50825703]\\n [ 0.75722361 -0.7008909  -2.10820389]]\\n...\\n199 5.73203 [[ 0.80269563  0.67861295 -1.21728313]\\n [-0.3051686  -0.3032113   1.50825703]\\n [ 0.75722361 -0.7008909  -2.10820389]]\\n200 5.73203 [[ 0.80269563  0.67861295 -1.21728313]\\n [-0.3051686  -0.3032113   1.50825703]\\n [ 0.75722361 -0.7008909  -2.10820389]]\\nPrediction: [0 0 0]\\nAccuracy:  0.0\\n-------------------------------------------------\\nWhen lr = 0.1\\n\\n0 5.73203 [[ 0.72881663  0.71536207 -1.18015325]\\n [-0.57753736 -0.12988332  1.60729778]\\n [ 0.48373488 -0.51433605 -2.02127004]]\\n1 3.318 [[ 0.66219079  0.74796319 -1.14612854]\\n [-0.81948912  0.03000021  1.68936598]\\n [ 0.23214608 -0.33772916 -1.94628811]]\\n...\\n199 0.672261 [[-1.15377033  0.28146935  1.13632679]\\n [ 0.37484586  0.18958236  0.33544877]\\n [-0.35609841 -0.43973011 -1.25604188]]\\n200 0.670909 [[-1.15885413  0.28058422  1.14229572]\\n [ 0.37609792  0.19073224  0.33304682]\\n [-0.35536593 -0.44033223 -1.2561723 ]]\\nPrediction: [2 2 2]\\nAccuracy:  1.0\\n'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lab 7 Learning rate and Evaluation\n",
    "from functools import partial as bind\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "assert(tf.__version__.find('2') == 0)\n",
    "tf.random.set_seed(777)  # for reproducibility\n",
    "\n",
    "x_data = tf.Variable([[1, 2, 1],\n",
    "          [1, 3, 2],\n",
    "          [1, 3, 4],\n",
    "          [1, 5, 5],\n",
    "          [1, 7, 5],\n",
    "          [1, 2, 5],\n",
    "          [1, 6, 6],\n",
    "          [1, 7, 7]], dtype=tf.float32)\n",
    "y_data = tf.Variable([[0, 0, 1],\n",
    "          [0, 0, 1],\n",
    "          [0, 0, 1],\n",
    "          [0, 1, 0],\n",
    "          [0, 1, 0],\n",
    "          [0, 1, 0],\n",
    "          [1, 0, 0],\n",
    "          [1, 0, 0]], dtype=tf.float32)\n",
    "\n",
    "# Evaluation our model using this test dataset\n",
    "x_test = tf.Variable([[2, 1, 1],\n",
    "          [3, 1, 2],\n",
    "          [3, 3, 4]], dtype=tf.float32)\n",
    "y_test = tf.Variable([[0, 0, 1],\n",
    "          [0, 0, 1],\n",
    "          [0, 0, 1]], dtype=tf.float32)\n",
    "\n",
    "# Try to change learning_rate to small numbers\n",
    "for lr in [1.5, 1e-10, 0.1]:\n",
    "    W = tf.Variable(tf.random.normal([3, 3]))\n",
    "    b = tf.Variable(tf.random.normal([3]))\n",
    "\n",
    "    # tf.nn.softmax computes softmax activations\n",
    "    # softmax = exp(logits) / reduce_sum(exp(logits), dim)\n",
    "    hypothesis = lambda X: tf.nn.softmax(tf.matmul(X, W) + b)\n",
    "\n",
    "    # Cross entropy cost/loss\n",
    "    @tf.function\n",
    "    def cost(X, Y):\n",
    "        return tf.reduce_mean(-tf.reduce_sum(Y * tf.math.log(hypothesis(X)), axis=1))\n",
    "\n",
    "    tf.print(\"When lr:\", lr)\n",
    "    optimizer = tf.keras.optimizers.SGD(learning_rate=lr)\n",
    "\n",
    "    # Correct prediction Test model\n",
    "    prediction = lambda X: tf.argmax(hypothesis(X), 1)\n",
    "    is_correct = lambda X, Y: tf.equal(prediction(X), tf.argmax(Y, 1))\n",
    "    accuracy = lambda X, Y: tf.reduce_mean(tf.cast(is_correct(X, Y), tf.float32))\n",
    "\n",
    "    # Launch graph\n",
    "    for step in range(201):\n",
    "        # Minimize\n",
    "        optimizer.minimize(bind(cost, x_data, y_data), var_list=[W, b])\n",
    "\n",
    "        if step % 20 == 0:\n",
    "            # evaluate training accuracy\n",
    "            tf.print(\"Step: \", step, \"cost: \", bind(cost, x_data, y_data)(), \"W: \", W)\n",
    "\n",
    "    # predict\n",
    "    tf.print(\"Prediction:\", prediction(x_test))\n",
    "    # Calculate the accuracy\n",
    "    tf.print(\"Accuracy: \", accuracy(x_test, y_test))\n",
    "    tf.print(\"-------------------------------------------------\")\n",
    "\n",
    "'''\n",
    "when lr = 1.5\n",
    "\n",
    "0 5.73203 [[-0.30548954  1.22985029 -0.66033536]\n",
    " [-4.39069986  2.29670858  2.99386835]\n",
    " [-3.34510708  2.09743214 -0.80419564]]\n",
    "1 23.1494 [[ 0.06951046  0.29449689 -0.0999819 ]\n",
    " [-1.95319986 -1.63627958  4.48935604]\n",
    " [-0.90760708 -1.65020132  0.50593793]]\n",
    "2 27.2798 [[ 0.44451016  0.85699677 -1.03748143]\n",
    " [ 0.48429942  0.98872018 -0.57314301]\n",
    " [ 1.52989244  1.16229868 -4.74406147]]\n",
    "3 8.668 [[ 0.12396193  0.61504567 -0.47498202]\n",
    " [ 0.22003263 -0.2470119   0.9268558 ]\n",
    " [ 0.96035379  0.41933775 -3.43156195]]\n",
    "4 5.77111 [[-0.9524312   1.13037777  0.08607888]\n",
    " [-3.78651619  2.26245379  2.42393875]\n",
    " [-3.07170963  3.14037919 -2.12054014]]\n",
    "5 inf [[ nan  nan  nan]\n",
    " [ nan  nan  nan]\n",
    " [ nan  nan  nan]]\n",
    "6 nan [[ nan  nan  nan]\n",
    " [ nan  nan  nan]\n",
    " [ nan  nan  nan]]\n",
    " ...\n",
    "Prediction: [0 0 0]\n",
    "Accuracy:  0.0\n",
    "-------------------------------------------------\n",
    "When lr = 1e-10\n",
    "\n",
    "0 5.73203 [[ 0.80269563  0.67861295 -1.21728313]\n",
    " [-0.3051686  -0.3032113   1.50825703]\n",
    " [ 0.75722361 -0.7008909  -2.10820389]]\n",
    "1 5.73203 [[ 0.80269563  0.67861295 -1.21728313]\n",
    " [-0.3051686  -0.3032113   1.50825703]\n",
    " [ 0.75722361 -0.7008909  -2.10820389]]\n",
    "...\n",
    "199 5.73203 [[ 0.80269563  0.67861295 -1.21728313]\n",
    " [-0.3051686  -0.3032113   1.50825703]\n",
    " [ 0.75722361 -0.7008909  -2.10820389]]\n",
    "200 5.73203 [[ 0.80269563  0.67861295 -1.21728313]\n",
    " [-0.3051686  -0.3032113   1.50825703]\n",
    " [ 0.75722361 -0.7008909  -2.10820389]]\n",
    "Prediction: [0 0 0]\n",
    "Accuracy:  0.0\n",
    "-------------------------------------------------\n",
    "When lr = 0.1\n",
    "\n",
    "0 5.73203 [[ 0.72881663  0.71536207 -1.18015325]\n",
    " [-0.57753736 -0.12988332  1.60729778]\n",
    " [ 0.48373488 -0.51433605 -2.02127004]]\n",
    "1 3.318 [[ 0.66219079  0.74796319 -1.14612854]\n",
    " [-0.81948912  0.03000021  1.68936598]\n",
    " [ 0.23214608 -0.33772916 -1.94628811]]\n",
    "...\n",
    "199 0.672261 [[-1.15377033  0.28146935  1.13632679]\n",
    " [ 0.37484586  0.18958236  0.33544877]\n",
    " [-0.35609841 -0.43973011 -1.25604188]]\n",
    "200 0.670909 [[-1.15885413  0.28058422  1.14229572]\n",
    " [ 0.37609792  0.19073224  0.33304682]\n",
    " [-0.35536593 -0.44033223 -1.2561723 ]]\n",
    "Prediction: [2 2 2]\n",
    "Accuracy:  1.0\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}