TensorFlow - 00 Basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from glob import glob\n",
    "\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hw:  Tensor(\"add:0\", shape=(), dtype=string)\n",
      "b'HelloWorld' <class 'bytes'> HelloWorld\n"
     ]
    }
   ],
   "source": [
    "h = tf.constant(\"Hello\")\n",
    "w = tf.constant(\"World\")\n",
    "hw = h + w\n",
    "print(\"hw: \", hw)\n",
    "with tf.Session() as session:\n",
    "    ans = session.run(hw)\n",
    "    print(ans, type(ans), ans.decode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f:  -3\n",
      "tf.Graph() =  <tensorflow.python.framework.ops.Graph object at 0x62ba3f0d0>\n",
      "tf.get_default_graph() =  <tensorflow.python.framework.ops.Graph object at 0x62b98a350>\n",
      "f.graph =  <tensorflow.python.framework.ops.Graph object at 0x62b98a350>\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "a = tf.constant(5, name = \"a\")\n",
    "b = tf.constant(4)\n",
    "c = tf.constant(3)\n",
    "\n",
    "d = tf.multiply(a, b)\n",
    "e = tf.add(c, d)\n",
    "f = tf.subtract(d, e)\n",
    "with tf.Session() as session:\n",
    "    print(\"f: \", session.run(f))\n",
    "    print(\"tf.Graph() = \", tf.Graph())\n",
    "    print(\"tf.get_default_graph() = \", tf.get_default_graph())\n",
    "    print(\"f.graph = \", f.graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'a:0' shape=() dtype=int32>,\n",
       " <tf.Tensor 'Const:0' shape=() dtype=int32>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# The name of the Tensor object is simply the name of its corresponding operation \n",
    "# (concatenated with a colon), followed by the index of that tensor in the outputs of the \n",
    "# operation that produced it —it is possible to have more than one.\n",
    "# Objects residing within the same graph cannot have the same name\n",
    "# As a consequence, it will automatically add an underscore and a number to distinguish the two. \n",
    "# However, both objects can have the same name when they are associated with different graphs\n",
    "\n",
    "a, b "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "False True\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "g1 = tf.get_default_graph()\n",
    "g2 = tf.Graph()\n",
    "print(g1 is tf.get_default_graph())\n",
    "with g2.as_default():\n",
    "    print(g1 is tf.get_default_graph(), g2 is tf.get_default_graph())\n",
    "print(g1 is tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expression c1 + c2 is not valid since these two variables belong to different graphs\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "g1 = tf.Graph()\n",
    "g2 = tf.Graph()\n",
    "with g1.as_default():\n",
    "    c1 = tf.constant(4.0)\n",
    "\n",
    "with g2.as_default():\n",
    "    c2 = tf.constant(2.0)\n",
    "\n",
    "try:\n",
    "    c = c1 + c2\n",
    "    with tf.Session() as session:\n",
    "        session.run(c)\n",
    "except ValueError as ex:\n",
    "    print(\"Expression c1 + c2 is not valid since these two variables belong to different graphs\")\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor '3/Const:0' shape=() dtype=float32>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Group variable with name scope\n",
    "with tf.name_scope(\"3\"):\n",
    "    c1 = tf.constant(4.0)\n",
    "c1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]\n"
     ]
    }
   ],
   "source": [
    "a = tf.constant(1.0)\n",
    "b = 1.0 + a\n",
    "c = 1.0 + b\n",
    "d = 1.0 + c\n",
    "e = 1.0 + d\n",
    "f = 1.0 + e\n",
    "\n",
    "fetchers = [a, b, c, d, e, f]\n",
    "\n",
    "with tf.Session() as session:\n",
    "    print(session.run(fetchers))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Default data types for tensorflow is int32 and float32 respectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Const_4:0' shape=(2, 3) dtype=int32>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.constant([[1, 2, 3], [4, 5, 6]])\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Const_7:0' shape=(2, 3) dtype=int64>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([[1, 2, 3], [4, 5, 6]])\n",
    "c = tf.constant(a)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('int64')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tf.float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.constant(4, dtype='float64')\n",
    "c.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Primitive Data Types\n",
    "- DT_FLOAT\ttf.float32\t32 bits floating point.\n",
    "- DT_DOUBLE\ttf.float64\t64 bits floating point.\n",
    "- DT_INT8\ttf.int8\t8 bits signed integer.\n",
    "- DT_INT16\ttf.int16\t16 bits signed integer.\n",
    "- DT_INT32\ttf.int32\t32 bits signed integer.\n",
    "- DT_INT64\ttf.int64\t64 bits signed integer.\n",
    "- DT_UINT8\ttf.uint8\t8 bits unsigned integer.\n",
    "- DT_UINT16\ttf.uint16\t16 bits unsigned integer.\n",
    "- DT_STRING\ttf.string\tVariable length byte arrays. Each element of a Tensor is a byte array.\n",
    "- DT_BOOL\ttf.bool\tBoolean.\n",
    "- DT_COMPLEX64\ttf.complex64\tComplex number made of two 32 bits floating points: real and imaginary parts.\n",
    "- DT_COMPLEX128\ttf.complex128\tComplex number made of two 64 bits floating points: real and imaginary parts.\n",
    "- DT_QINT8\ttf.qint8\t8 bits signed integer used in quantized Ops.\n",
    "- DT_QINT32\ttf.qint32\t32 bits signed integer used in quantized Ops.\n",
    "- DT_QUINT8\ttf.quint8\t8 bits unsigned integer used in quantized Ops.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TensorFlow operation\tDescription\n",
    "- tf.constant(value)\tCreates a tensor populated with values of as specified by arguments “value”\n",
    "- tf.fill(shape,value)\tCreates a tensor of shape “shape\" and fills it with “value\" \n",
    "- tf.zeros(shape)\tReturns a tensor of shape “shape” and all elements set to zero\n",
    "- tf.zeros_like(tensor)\tReturns a tensor of the same type and shape as “tensor\" with all elements set to zero\n",
    "- tf.ones(shape)\tReturns a tensor of shape “shape\" and all elements set to 1\n",
    "- tf.ones_like(tensor)\tReturns a tensor of the same type and shape as “tensor\" with all elements set to 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 1., 1.],\n",
       "       [1., 1., 1.],\n",
       "       [1., 1., 1.],\n",
       "       [1., 1., 1.]], dtype=float32)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.ones((4, 3))\n",
    "session.run(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 1],\n",
       "       [1, 1, 1],\n",
       "       [1, 1, 1],\n",
       "       [1, 1, 1]], dtype=int32)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.fill((4, 3), 1)\n",
    "session.run(c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TensorFlow operation\tDescription\n",
    "- tf.random_normal(shape, mean, stddev)\tOutputs random values from a normal distribution\n",
    "- tf.truncated_normal(shape, mean, stddev)\tOutputs random values from a truncated normal distribution (values whose  magnitude is more than 2 standard deviations from the mean are dropped and re-picked).\n",
    "- tf.random_uniform(shape, minval, maxval)\tGenerated values from a uniform distribution in the range [minval, maxval)\n",
    "- tf.random_shuffle(tensor)\tRandomly shuffles a tensor along its first dimension\n",
    "- tf.random_crop(tensor, shape)\tSlices a shape “shape” portion out of “tensor\" at a uniformly chosen offset\n",
    "- tf.multinomial(logits, n_samples)\tDraws samples from a multinomial distribution\n",
    "- tf.random_gamma(shape,alpha,beta)\tDraws “shape\" samples from each of the given Gamma distribution(s)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.9617498 , 0.44918132, 0.5127076 ],\n",
       "       [0.8348917 , 0.54734325, 0.8099698 ],\n",
       "       [0.71889913, 0.39656377, 0.9537337 ],\n",
       "       [0.93378425, 0.04777336, 0.07103348]], dtype=float32)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.random_uniform((4, 3), seed=100)\n",
    "session.run(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.026301  , -0.07955144, -0.29859897],\n",
       "       [ 0.1763167 , -0.30626023,  0.12118966],\n",
       "       [ 0.14748672, -0.19404596, -0.03731908],\n",
       "       [ 0.08446319,  0.3193688 ,  0.66740936]], dtype=float32)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.random_normal((4, 3), mean=0, stddev=0.3, seed=100)\n",
    "session.run(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([0.2       , 0.22500001, 0.25      , 0.275     , 0.3       ],\n",
       "       dtype=float32),)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.linspace(0.2, 0.3, 5)\n",
    "print(c.get_shape())\n",
    "session.run(c), "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.4 , 0.65, 0.9 , 1.15, 1.4 , 1.65, 1.9 , 2.15, 2.4 , 2.65, 2.9 ,\n",
       "       3.15, 3.4 , 3.65, 3.9 , 4.15, 4.4 , 4.65, 4.9 ], dtype=float32)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.range(0.4, 5.0, 0.25)\n",
    "print(c.get_shape())\n",
    "session.run(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.9999913 ],\n",
       "       [0.14483425],\n",
       "       [0.9941877 ],\n",
       "       [0.55156505],\n",
       "       [0.9913591 ]], dtype=float32)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = tf.random_normal((5, 10), mean = 0.0, stddev = 1.0, seed = 0)\n",
    "w = tf.random_normal((10, 1), mean = 0.0, stddev = 1.0, seed = 0)\n",
    "b = tf.fill((5, 1), 1.0) # Create tensors of a given value\n",
    "s = tf.sigmoid(tf.matmul(X, w) - b)\n",
    "session.run(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TensorFlow operator\tShortcut\tDescription\n",
    "- tf.add()\ta + b\tAdd a and b, element-wise\n",
    "- tf.mul()\ta * b\tMultiply a and b, element-wise\n",
    "- tf.sub()\ta - b\tSubtract a from b, element-wise\n",
    "- tf.div()\ta / b\tWill perform element-wise integer division when given an integer type tensor, and floatingpoint (“true”) division on floating point tensors\n",
    "- tf.pow()\ta ** b\tThe result of raising each element in a to its corresponding element b, element-wise\n",
    "- tf.mod()\ta % b\tElement-wise modulo\n",
    "- tf.logical_and()\ta & b\tReturns the truth table of a & b, element-wise. dtype must be tf.bool\n",
    "- tf.greater()\ta > b\tReturns the truth table of a > b, element-wise\n",
    "- tf.greater_equal()\ta >= b\tReturns the truth table of a >= b, element-wise\n",
    "- tf.less_equal()\ta <= b\tReturns the truth table of a <= b , element-wise\n",
    "- tf.less()\ta < b\tReturns the truth table of a < b, element-wise\n",
    "- tf.neg()\t-a\tReturns the negative value of each element in a\n",
    "- tf.logical_not()\t~a\tReturns the logical NOT of each element in a. Only compatible with Tensor objectswith dtype of - tf.bool\n",
    "- tf.abs()\tabs(a)\tReturns the absolute value of each element in a\n",
    "- tf.logical_or()\ta | b\tReturns the truth table of a | b, element-wise. dtype must be tf.bool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Variable \n",
    "\n",
    "During the optimization process, tuning the weights of the model by iterative updates requires that their current state be maintained. For that purpose, TensorFlow uses special objects called “Variables”. Variables, unlike other Tensor objects that are “refilled” across calls to run(), can maintain a fixed state in the graph. Like other Tensors, Variables can be used as input for other operations in the graph.\n",
    "\n",
    "Using Variables is done in two stages: (1) we call the tf.Variable() function in order to create a Variable, and define what value it will be initialized with and (2) we then have to explicitly perform an initialization operation by running the session with the tf.initialize_all_variables() method, which allocates the memory for the variable and sets the initial value as defined in (1).\n",
    "\n",
    "Like with other Tensor objects, Variables are only computed when the model runs, as we can see in the following example:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = tf.constant(15, name = \"c\")\n",
    "x = tf.Variable(c * 5, name = \"x\")\n",
    "init = tf.global_variables_initializer()\n",
    "session.run(init)\n",
    "session.run(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Placeholder\n",
    "\n",
    "TensorFlow has designated built-in structures for feeding input values. These structures are called placeholders. Placeholders can be thought of as empty variables that will be filled with data later on. We use them by first constructing our graph and only when it is executed feed them with the input data.\n",
    "\n",
    "Placeholders have three arguments we are already familiarized with: dtype for the type of data that will be inserted to them and two optional arguments — shape and name. If a shape is not fed, then the placeholder can be fed with data of any size. This is also the same as placing ‘None’ for a shape. We can also place ‘None’ for specific dimensions we are not sure about their length —for example, it is very common to use only for the rows dimension of a matrix, corresponding to the number of samples, while having the length of the columns (features) fixed. We feed the input values when running the session, just after stating what outputs we want to evaluate.\n",
    "\n",
    "Whenever we define a placeholder, we must feed it with some input values or else an exception will be thrown. The input data is given as a dictionary, where each key corresponds to a placeholder variable name, and the matching values are the data values given in the form of a list or a numpy array.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.95023626],\n",
       "       [0.05953365],\n",
       "       [0.9869032 ],\n",
       "       [0.57299566],\n",
       "       [0.16911489]], dtype=float32)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_data = np.random.randn(5, 10)\n",
    "w_data = np.random.randn(10, 1)\n",
    "\n",
    "X = tf.placeholder(\"float32\", shape=(5, 10))\n",
    "w = tf.placeholder(\"float32\", shape=(10, 1))\n",
    "b = tf.fill((5, 1), - 1.0)\n",
    "s = tf.sigmoid(tf.matmul(X, w) + b)\n",
    "\n",
    "session.run(s, feed_dict={X: X_data, w: w_data})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.4"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = tf.constant([3, 4, 5, 9, 11], dtype=\"float32\")\n",
    "mean = tf.reduce_mean(r)\n",
    "session.run(mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint a tensorflow session\n",
    "\n",
    "You can use checkpoint to save the model to the disk and reload it later during inferencing. In this way, during inferencing time, you do not need to retrain the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.0\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    W = tf.Variable(3.0, dtype=tf.float32, name = \"W\")\n",
    "    W = W.assign(W+1)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print(sess.run(W))\n",
    "    saver = tf.train.Saver()\n",
    "    saver.save(sess, \"/tmp/1/model.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restoring a tensorflow session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/1/model.ckpt\n",
      "4.0\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "W1 = tf.Variable(3.0, dtype=tf.float32, name = \"W\")\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, \"/tmp/1/model.ckpt\")\n",
    "    print(sess.run(W1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((30, 2), (30,))"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_data = np.random.ranf((10 * 3, 2))\n",
    "X_data = (X_data - X_data.mean(axis = 0)) / X_data.std(axis=0)\n",
    "W_true = np.array([0.1, 5.9])\n",
    "b_true = 1.4\n",
    "y_data = X_data.dot(W_true) + b_true\n",
    "X_data.shape, y_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_pred Tensor(\"add:0\", shape=(?, 1), dtype=float32)\n",
      "dW: Tensor(\"gradients/MatMul_grad/MatMul_1:0\", shape=(2, 1), dtype=float32)\n",
      "db Tensor(\"gradients/add_grad/Reshape_1:0\", shape=(), dtype=float32)\n",
      "[[0.10000442]\n",
      " [5.899987  ]] 1.3999972\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHhxJREFUeJzt3Xl0HfWZ5vHvq12yJMtavMpGNjbGxsQBhLENTQgQFich6YR0QzrpEDLjMydknT5JIJ0eOj0zPTQ9k046mcmEDlkmndjdYQmEgCFhCSFhkw3BG8YLxpbkRZIXWcaytnf+qCtLlmXrSrpXVXX1fM65p27VrVv1lgoe/+6vNnN3REQkPrLCLkBERIZHwS0iEjMKbhGRmFFwi4jEjIJbRCRmFNwiIjGj4BYRiRkFt4hIzCi4RURiJicdC62srPSampp0LFpEJCOtXbu22d2rkpk3LcFdU1NDXV1dOhYtIpKRzOytZOdVV4mISMwkFdxmVmZm95nZ62a22cyWpbswEREZXLJdJd8C1rj7jWaWBxSlsSYRETmDIYPbzEqBy4FbANy9A+hIb1kiInI6yXSVzAGagB+a2Stm9n0zm5DmukRE5DSSCe4c4ELgu+5+AXAUuH3gTGa20szqzKyuqakpxWWKiEivZIK7Hqh39xcT4/cRBPlJ3P0ed69199qqqqRORRQRkREYso/b3fea2W4zm+/uW4CrgE3pL01ixx26O6C7E3q6oKc7Mez/GmzagM+9J/HyYIifPO49ifUNNU//ad5vPDGtr/BTt+PkCWf4fBjfHXK5nP7zYdU06MLiI86PU8ybAJd9Ie2rSfasks8CP02cUbID+GT6SpK0cofjrfD2ATh2AI63QUdbYnjk1PGOo9DZDt3Hoes4dLX3G3acPN59POytk4xhYRcwMsWToxPc7v4qUJvmWmSkesO4tTF4HdnT9/5oU19I9w57uoZYoEF+CeQVBy2I3ELIKYCcfCic1Pc+Oz8Y9o7n5EN2HmTnQlYOZOVCVnbife9r4HhiWu93LBuysoIaLAssMTwxPnCaJTFPv3Gzvu/1DwcbGBQDxs/0+bC+O4zlDve7pyxLMlVaLnmXNDnaAs1boGU7HNgBB7ZDyw44+GbQSh6oqAImTIaicqicB4XlwbSixLBwUl9AnxgWQ26RQkAkwhTcUXW4HhrWwt71fa/Whr7Ps3JgUg2Uz4GaS6F0BpROD14l04JXbkFo5YtI+ii4o8AdWrbBzt/BW8/Drufh8O7gM8uGqvlQcxlMPR+qFkDFHJg4C7K1+0TGI/2fH5au47DzOXjjcdj6OBzcGUwvngKzlsHyz0L1xTB5oVrOInISBfdY6ukOWtWv/Rw2PxwcUMwphNmXw/LPwZwrgq4P9S+LyBkouMdC236o+yGs/REcaYS8Elh4Ayz8IMz+k+CsDRGRJCm402nvevjDt2HDA9DTCXOvhmv/O8y/XmEtIiOm4E6HfZvgmf8RdIfklUDtrbBkJVTODbsyEckACu5UOrIPfnMn/HF1cE70u74CSz8NhWVhVyYiGUTBnQrdXfDyv8DTfw+dx+DSz8GlXwgudBERSTEF92i1bIcH/xPUvwRnXwXX360uERFJKwX3SLnDuh/Dmq8GF8J8+F5Y9GGdyiciaafgHonOdnjki/DHn8Hsd8EHvwsTZ4RdlYiMEwru4WpthNV/AY3r4F23Bwcgs5J5HoWISGoouIejeRv85E+DW6P++U9hwfvCrkhExiEFd7L2/BF+8iHA4ZZfwfR3hl2RiIxTCu5k7F0PP35/cDHNxx+EqnPCrkhExjEF91BatgfdI3nF8MlHYdJZYVckIuOcjqqdSWsj/L8PBA+X/fgvFNoiEglqcZ9Ox9uw6mY4dghueUTdIyISGQruwbjDw58JDkjevFoHIkUkUtRVMpjnvwMb7oer/gbmXxd2NSIiJ1FwD9SwDn7zt3Du++Cy/xx2NSIip1Bw93e8De7/D8FzH2/4tu47IiKRpD7u/h7/KhzYERyM1C1ZRSSikgpuM9sJHAG6gS53r01nUaHY+Vxwt7/ln4Way8KuRkTktIbT4n63uzenrZIwdR2HX34Bys6CK+4IuxoRkTNSVwnA774BLVvhY/dD3oSwqxEROaNkD0468ISZrTWzlYPNYGYrzazOzOqamppSV2G6HdoNv/9m8BCEuVeHXY2IyJCSDe5L3f1C4HrgNjO7fOAM7n6Pu9e6e21VVVVKi0yrp/5bcMHN1V8PuxIRkaQkFdzu3pgY7gceBJaks6gx0/gqvLYaln0aymaGXY2ISFKGDG4zm2BmJb3vgWuADekubEw88TUoqoDLvhh2JSIiSUvm4OQU4EELLkbJAX7m7mvSWtVY2Pl72Pk7uO4uKJgYdjUiIkkbMrjdfQeweAxqGVvP/iNMmAwX3RJ2JSIiwzI+L3mvr4MdTwcX2+QWhl2NiMiwjM/gfvYfobAcam8NuxIRkWEbf8G9/3V4Yw0s/TTkF4ddjYjIsI2/4H7pHsjOV2tbRGJrfAV3+2H442o4/yMwoSLsakRERmR8BfcrP4XOo7DkP4ZdiYjIiI2f4O7pgZf/BWZeomdIikisjZ/gfvOZ4CEJSwa9R5aISGyMn+B+9WdQUBY8S1JEJMbGR3C3H4bNv4Tzb4TcgrCrEREZlfER3BsfhK52WPzRsCsRERm18RHcr66Cyvkw48KwKxERGbXMD+6DO2H3C7D4JgjucCgiEmuZH9ybHgqGiz4Ubh0iIimS+cG98Rcw/QKYVBN2JSIiKZHZwX1wJzSug4UfDLsSEZGUyezg7u0mOU/BLSKZI/ODW90kIpJhMje4j+yDhrVw7nvDrkREJKUyN7i3PhEMz7k+3DpERFIsc4P7jTVQWg1Tzgu7EhGRlMrM4O46DtufhnOu0UU3IpJxMjO4dz4XPDDhnOvCrkREJOUyM7i3PgE5hTD78rArERFJuaSD28yyzewVM3sknQWlxPanoOZSyC0MuxIRkZQbTov788DmdBWSMq2N0PwGzLki7EpERNIiqeA2s2rgvcD301tOCuz4bTCc/a5w6xARSZNkW9zfBL4M9KSxltR487dQVAFTFoVdiYhIWgwZ3Gb2PmC/u68dYr6VZlZnZnVNTU0pK3BY3IMWd82fQFZmHncVEUkm3S4FbjCzncBq4Eoz+9eBM7n7Pe5e6+61VVVVKS4zSS3b4Eij+rdFJKMNGdzufoe7V7t7DXAT8JS7fyztlY3EjmeC4Rz1b4tI5sqs/oS3/gClM2DS7LArERFJm5zhzOzuzwDPpKWS0XKHXS/AWct0mbuIZLTMaXEf3h30b89cGnYlIiJplTnBveuFYDhLwS0imS2zgjuvRLdxFZGMl1nBPfNiyMoOuxIRkbTKjOA+dgj2b4JZy8KuREQk7TIjuOvrAIeZl4RdiYhI2mVGcDesBQxmXBh2JSIiaZcZwd24DqrmQ35J2JWIiKRd/IPbHRrWwfQLwq5ERGRMxD+4Wxvh6H6Yrm4SERkf4h/cjeuCoVrcIjJOxD+4G9ZBVg5MPT/sSkRExkT8g7vxFZi8EHILwq5ERGRMxDu43YPgVjeJiIwjw7qta+QcfBPaDym4RQSAzs5O6uvraW9vD7uU0yooKKC6uprc3NwRLyPewb13QzCctjjcOkQkEurr6ykpKaGmpgaL4H353Z2Wlhbq6+uZPXvkD3yJd1fJvg1gWTB5QdiViEgEtLe3U1FREcnQBjAzKioqRv2LIObBvREq5kJuYdiViEhERDW0e6WivngH9971uv+2iETOmjVrmD9/PnPnzuWuu+5K+fLjG9ztrXDoLZiyKOxKRERO6O7u5rbbbuOxxx5j06ZNrFq1ik2bNqV0HfEN7v2bg6GCW0Qi5KWXXmLu3LnMmTOHvLw8brrpJh566KGUriO+Z5XsWx8M1VUiIoP4+i83sqmxNaXLXDi9lDvff+bMaWhoYObMmSfGq6urefHFF1NaR3xb3Ps2QsFEmFgddiUiIie4+ynTUn3ANL4t7r0bgm6SiB9BFpFwDNUyTpfq6mp27959Yry+vp7p06endB3xbHG7B8+YVDeJiETMxRdfzNatW3nzzTfp6Ohg9erV3HDDDSldx5AtbjMrAJ4F8hPz3+fud6a0iuE6XA8dbbrwRkQiJycnh+985ztce+21dHd3c+utt3LeealtZCbTVXIcuNLd28wsF3jOzB5z9xdSWslwNG8JhpXzQytBROR0VqxYwYoVK9K2/CGD24Oe9rbEaG7idWrv+1hqeiMYVim4RWT8SaqP28yyzexVYD/wa3c/5dwWM1tpZnVmVtfU1JTqOk/WvAUKy2FCZXrXIyISQUkFt7t3u/s7gWpgiZmdctWLu9/j7rXuXltVVZXqOk/WtEWtbREZt4Z1Vom7HwKeAa5LSzXJatoCleeEWoKISFiGDG4zqzKzssT7QuBq4PV0F3ZaR5vh2AG1uEVk3ErmrJJpwI/NLJsg6P/d3R9Jb1ln0KQzSkRkfBuyxe3ur7n7Be7+Dndf5O5/NxaFnVbvqYBV6ioRkei59dZbmTx5MosWpe8GePG7crLpDcgtglLdo0REoueWW25hzZo1aV1H/IK7eQtUzoOs+JUuIpnv8ssvp7y8PK3riN9Nplq2QfWSsKsQkah77PbgKVmpNPV8uD71T7QZrng1W7uOB/cpqTg77EpEREITrxb3wbfAe6BcwS0iQ4hAyzhd4tXiPrAjGJbPCbcOEZEQKbhFRFLo5ptvZtmyZWzZsoXq6mruvffelK8jXl0lB3ZA/kQoSu8RWxGRkVq1alXa1xG/Fnf5bD2uTETGtfgFt84oEZFxLj7B3d0Jh3apf1tExr34BPehXeDdCm4ROaPgoV3RlYr64hPcOqNERIZQUFBAS0tLZMPb3WlpaaGgoGBUy4nPWSUKbhEZQnV1NfX19aT98YmjUFBQQHX16G6SF6/gziuBCWl+LJqIxFZubi6zZ88Ou4y0i09XycGdMKlGpwKKyLgXn+A+tAvKZoVdhYhI6OIR3O4KbhGRhHgE97GD0NGm4BYRIS7BfWhXMFRwi4jELbhnhluHiEgExCy41eIWEYlPcOeXQkFZ2JWIiIQuPsFdNkvncIuIkERwm9lMM3vazDab2UYz+/xYFHYSnQooInJCMi3uLuCv3H0BsBS4zcwWpresfnrP4Z6oA5MiIpBEcLv7Hndfl3h/BNgMzEh3YSccOwgdR9TiFhFJGFYft5nVABcAL6ajmEEd3h0MFdwiIsAwgtvMioH7gS+4e+sgn680szozq0vpLRV1KqCIyEmSCm4zyyUI7Z+6+wODzePu97h7rbvXVlWl8NarCm4RkZMkc1aJAfcCm939G+kvaYDDDZBbBIWTxnzVIiJRlEyL+1Lg48CVZvZq4rUizXX1aW2A0uk6h1tEJGHIJ+C4+3NAeKnZ2hgEt4iIAHG4crK1EUrH7uxDEZGoi3Zw93TDkT1qcYuI9BPt4G7bD96t4BYR6Sfawd3aGAzVVSIickLEg7shGKrFLSJyQsSDWy1uEZGBIh7cDZCdB0UVYVciIhIZEQ/uRl18IyIyQAyCW90kIiL9RTy4G3RgUkRkgOgGt7suvhERGUR0g/vtFujuUFeJiMgA0Q1uncMtIjKoCAd34hzuEgW3iEh/0Q3uI3uDYcnUcOsQEYmY6AZ3275gOCGFj0ETEckA0Q7uogrIyQu7EhGRSIlucB/ZB8XqJhERGSi6wd22D4onh12FiEjkRDu4dWBSROQU0Qxud7W4RUROI5rBfexgcNWk+rhFRE4RzeDuPRVQLW4RkVNEO7jVxy0icopoBveR3ha3gltEZKAhg9vMfmBm+81sw1gUBKirRETkDJJpcf8IuC7NdZysbR/kFkF+yZiuVkQkDoYMbnd/FjgwBrX06T0VUM+aFBE5Rcr6uM1spZnVmVldU1PT6BZ2ZK/6t0VETiNlwe3u97h7rbvXVlWN8o5+bfugZEpqChMRyTDRPKukbR8UK7hFRAYTveDuPAbthxXcIiKnkczpgKuA54H5ZlZvZp9Ka0Vt+4OhgltEZFA5Q83g7jePRSEnHG0OhnryjYjIoKLXVfK2gltE5EyiF9xHE6cSTqgItw4RkYiKYHCrxS0iciYRDO6m4HL3vAlhVyIiEklDHpwcc2+3QFFlUrO6O+t2HeLXm/axaU8r+w6309ndQ2lhLlNK81k4bSLvmDmRS2aXU5QXvU0VERmJ6KXZ0SaYMHRwP/X6Pv7hsS1s2XeE3Gxj3uQSzqooIi8ni8PHOtm6v40nNu3DHfJyslh+dgXXL5rKivOnUVKQOwYbIiKSHtEM7jOcw93e2c2dD23k3+p2M7tyAnff+A6uXzR10DBuO97Fq7sO8fSW/fxm8z6+cv96/vbhTVy/aCp/dvFMLpldjulGViISMxEM7haYsmjwj453sfIndfx+WwufvuJsvviec8jNPn03fXF+DpfNq+SyeZV87b0LeGX3IX5eV88jf2zkgVcaOHdqCbcsr+ED75xBYV52urZIRCSlohXc7kGLu+jUUwG7e5zPrXqF57e38L8+spgPX1Q9rEWbGRfOmsSFsybxX963kF++1sgPf7+T2x9Yz11rXufmJbP4+NKzmF5WmKqtERFJi2gFd0cbdB8f9FTAb/x6C0++vp//+sFFww7tgQrzsvmz2pl85KJqXnzzAD/6/U6+99vt3PPsDq47byqfWF7DxTWT1I0iIpEUreA+cfHNyQcn1751gP/zzHb+vHYmH196VspWZ2YsnVPB0jkV7D7wNj954S1Wv7SLX63fw8JppdxyaQ03LJ5OQa66UUQkOqJ1HvfRlmDYr8Xd2d3Dl+57jRllhfzN+xembdUzy4v46ooFvPDVq/j7Pz2f7h7ny/e9xvK7nuLuNa/TeOhY2tYtIjIckW9x/+zFXexoOsr3/7KW4vz0l1uUl8NHL5nFzUtm8vyOFn78h538399u53vP7uDa86bw0SVnsfzsCrKy1I0iIuGIVnD33mAqcQFO2/EuvvXkVpbNqeCqBWP7xHczY/nZlSw/u5LdB97mX194i9Uv7+bR9XuZUVbIhy+q5iMXVTOzvGhM6xIRiVZwD2hxr35pFweOdvCl6+aHeqBwZnkRd6xYwBffcw5PbNrHz+t28+2ntvLPT25l6ZxyPnRBNdecN4WyorzQahSR8SNiwd0CecWQW0hndw8/eO5Nlswu58JZk8KuDICC3GxuWDydGxZPp+HQMR5YW8996+r58v2v8dUHjeVzK1mxaCrXnDeV8gkKcRFJj4gFd9/l7o9v3Evj4Xb+7gODX4wTthllhXz2qnl85sq5bGho5Vfr9/Do+j3c/sB6/voXG7jorElcMb+Kd51TxcJppTq1UERSJlrB/Xbzif7tf6+rZ0ZZIe8+d2z7tofLzDi/eiLnV0/kK9fNZ2NjK49t2MMzW5q4e80W7l6zhaqSfC6fV8XSOeUsmV3OrPIiBbmIjFi0gvtoE5RW03DoGL/b2sRnr5xHdozO3jAzFs2YyKIZE/nSteeyv7WdZ7c289s3mnjy9X3cv64egMkl+VxcU05tzSQWzyxjwdRSXXIvIkmLWHA3w7TFPLiuHnf4yCivkAzb5NICbryomhsvqqanx9m6v42Xdx7g5Z0HqNt5kF+t3wNAlsHZVcWcN72URTMmsmBaKXMnFzO5JF8tcxE5RXSC2z0I7glV/Gr9Xi46a1JGnWqXlWXMn1rC/KklfCxx9WfjoWOsbzjMxobDbGxs5fkdLfzi1cYT3ynOz2F25QTOrprAnKpiaionMKOsgBllRVSV5Mfq14iIpE50gtsMbt/FrpYjbP5NHV9774KwK0q76WWFTC8r5Nrzpp6Y1tx2nNf3HGFHcxvb97exo/koL+88eFKgA+RkGVMnFjCjrJAZZYVMLi2gsjiPiuI8KovzqZiQT2VxHuUT8sg5wx0URSR+ohPcAHlFPPpG0H3QP8zGk8rifC6bl89l806+X8vbHV3UHzxGw8FjNBw6RuOhvuELO1pobuugo7tn0GWWFeUysTCXkoIcSgtyKS1IvC/se19ckENRXjaFucGroN/7wrxsChLvc7NN3TciIYtWcAO/3rSPRTNKM6qbJBWK8nI4Z0oJ50wpGfRzd6e1vYuWtuM0t3UEw6MdNB85zoGjHbS2d3KkvYvWY5282XyU1vZOWo91crSje1h1ZGcZ+TlZ5GQZeTlZ5Gb3vuyU93mJ+XKzs8hNvM+y4JWdRfA+y8g2I8s48T47K/jHITsLsq33fe90TswDwQFhI/jB1vvPiZn1jSf+kembx3onnTSNk6b1LZNBvnfqeo1+qxqx0f5zOLr1j27t4W/7KOsf5fp75eVkcfk56X/QeVLBbWbXAd8CsoHvu/td6Sjm8LFOXtl1kNvePTcdi89oZsbEwqBlPWcY/910dffQdryLI+1dtHd2c6yzm2MdwbBvvCcx7OLtjm46u3vo7HY6unvo7Oqhq6fvfWd3Yryrh6PHu+jsdjq7e+jo7qGnx+l2p6cHetzp7vF+Q/o+T8zT+949fX83kVSqLM6n7mtXp309Qwa3mWUD/xt4D1APvGxmD7v7plQX84dtzfQ4Y/IvlgRysrMoK8qL9OX63j/c+wV+jwMOTl+4e2L+YNg7zRPznTzNvW9++n+WWGbf+75l9s3X9/3+3x3xNjK6BYxm/XGuPRVSuf6xOmEgmRb3EmCbu+8AMLPVwAeAlAf3s1ubKc7P4Z0zy1K9aIkxMyMnW/3qIr2SOd1gBrC733h9YlpKuTvPvtHE8rMrzvgcSRGR8S6ZFvdgTZ1TflyY2UpgJcCsWbOGXcjxrh4unVvBpXMrh55ZRGQcSya464GZ/cargcaBM7n7PcA9ALW1tcPuNSrIzebuGxcP92siIuNOMn0SLwPzzGy2meUBNwEPp7csERE5nSFb3O7eZWafAR4nOB3wB+6+Me2ViYjIoJI6j9vdHwUeTXMtIiKSBJ2+ISISMwpuEZGYUXCLiMSMgltEJGYU3CIiMWOehju8mFkT8NYIv14JNKewnDjQNo8P2ubMN5rtPcvdk7rDXlqCezTMrM7da8OuYyxpm8cHbXPmG6vtVVeJiEjMKLhFRGImisF9T9gFhEDbPD5omzPfmGxv5Pq4RUTkzKLY4hYRkTOITHCb2XVmtsXMtpnZ7WHXkypmNtPMnjazzWa20cw+n5hebma/NrOtieGkxHQzs39O/B1eM7MLw92CkTOzbDN7xcweSYzPNrMXE9v8b4nbBGNm+YnxbYnPa8Kse6TMrMzM7jOz1xP7e1mm72cz+2Liv+sNZrbKzAoybT+b2Q/MbL+Zbeg3bdj71cw+kZh/q5l9YjQ1RSK4+z2Q+HpgIXCzmS0Mt6qU6QL+yt0XAEuB2xLbdjvwpLvPA55MjEPwN5iXeK0Evjv2JafM54HN/cb/AfinxDYfBD6VmP4p4KC7zwX+KTFfHH0LWOPu5wKLCbY9Y/ezmc0APgfUuvsigts+30Tm7ecfAdcNmDas/Wpm5cCdwCUEz/G9szfsR8TdQ38By4DH+43fAdwRdl1p2taHgPcAW4BpiWnTgC2J998Dbu43/4n54vQieFLSk8CVwCMEj8BrBnIG7nOCe70vS7zPScxnYW/DMLe3FHhzYN2ZvJ/pex5teWK/PQJcm4n7GagBNox0vwI3A9/rN/2k+Yb7ikSLmzF6IHHYEj8NLwBeBKa4+x6AxHByYrZM+Vt8E/gy0JMYrwAOuXtXYrz/dp3Y5sTnhxPzx8kcoAn4YaJ76PtmNoEM3s/u3gD8T2AXsIdgv60ls/dzr+Hu15Tu76gEd1IPJI4zMysG7ge+4O6tZ5p1kGmx+luY2fuA/e6+tv/kQWb1JD6LixzgQuC77n4BcJS+n8+Dif02J37qfwCYDUwHJhB0FQyUSft5KKfbxpRue1SCO6kHEseVmeUShPZP3f2BxOR9ZjYt8fk0YH9ieib8LS4FbjCzncBqgu6SbwJlZtb71KX+23VimxOfTwQOjGXBKVAP1Lv7i4nx+wiCPJP389XAm+7e5O6dwAPAcjJ7P/ca7n5N6f6OSnBn7AOJzcyAe4HN7v6Nfh89DPQeWf4EQd937/S/TBydXgoc7v1JFhfufoe7V7t7DcG+fMrd/wJ4GrgxMdvAbe79W9yYmD9WLTF33wvsNrP5iUlXAZvI4P1M0EWy1MyKEv+d925zxu7nfoa7Xx8HrjGzSYlfKtckpo1M2J3+/TrrVwBvANuBvw67nhRu12UEP4leA15NvFYQ9O09CWxNDMsT8xvBGTbbgfUER+xD345RbP8VwCOJ93OAl4BtwM+B/MT0gsT4tsTnc8Kue4Tb+k6gLrGvfwFMyvT9DHwdeB3YAPwEyM+0/QysIujD7yRoOX9qJPsVuDWx7duAT46mJl05KSISM1HpKhERkSQpuEVEYkbBLSISMwpuEZGYUXCLiMSMgltEJGYU3CIiMaPgFhGJmf8P0vxfMqE1ebIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a2fa181d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(1)\n",
    "\n",
    "X = tf.placeholder(\"float32\", (None, 2))\n",
    "y = tf.placeholder(\"float32\", (None, 1))\n",
    "learning_rate = tf.constant(0.01)\n",
    "\n",
    "W = tf.Variable(tf.truncated_normal((2, 1), stddev=0.1))\n",
    "b = tf.Variable(0.0)\n",
    "\n",
    "y_pred = tf.matmul(X, W) + b\n",
    "print(\"y_pred\", y_pred)\n",
    "mse = tf.reduce_mean((y - y_pred) ** 2)\n",
    "\n",
    "dW, db = tf.gradients(mse, [W, b])\n",
    "print(\"dW:\", dW)\n",
    "print(\"db\", db)\n",
    "\n",
    "grad_updates = tf.group([\n",
    "    W.assign_sub(learning_rate * dW),\n",
    "    b.assign_sub(learning_rate * db)\n",
    "])\n",
    "\n",
    "history = {\"mse\": [], \"W\": [], \"b\": []}\n",
    "values = {X: X_data, y: y_data.reshape(-1, 1)}\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    \n",
    "    sess.run(tf.global_variables_initializer())    \n",
    "    for i in range(1000):\n",
    "        mse_, W_, b_, _ = sess.run([mse, W, b, grad_updates], feed_dict= values)\n",
    "        history[\"mse\"].append(mse_)\n",
    "        history[\"W\"].append(W_)\n",
    "        history[\"b\"].append(b_)\n",
    "    pd.DataFrame(np.array(history[\"W\"])[:,:,0]).plot()\n",
    "    print(W_, b_)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1a302c5588>"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEKCAYAAAARnO4WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGv9JREFUeJzt3X+U3HV97/HXe2Z/ZMnml5sACUkI+SWSHATcewM3EhW0TbGkeuDcg2A99Eq31wq2vVagcltQT2lBj16vUltMrXq6alu4FtTgjxp+KsllgyEmEEwIBEMQwhKSbNhM9se7f+zMMruZH9/Zne+PmXk+zsk5u7Pf3e+bcX193/P+fOa75u4CANS/VNwFAACiQeADQIMg8AGgQRD4ANAgCHwAaBAEPgA0CAIfABoEgQ8ADYLAB4AG0RR3Aflmz57tixYtirsMAKgZW7ZsecXd5wQ5NlGBv2jRIvX09MRdBgDUDDPbG/RYRjoA0CAIfABoEAQ+ADQIAh8AGkSogW9mM83sLjPbaWZPmdkFYZ4PAFBc2Lt0vijph+5+uZm1SDop5PMBAIoILfDNbLqkNZKuliR3Py7peFjnAwCUFuZIZ7GkA5L+ycx+YWbrzWxqiOcDAJQQZuA3STpP0lfc/VxJRyXdOP4gM+sysx4z6zlw4ECI5SRUd7f2zThZw5bSvhkn62OX/rkW3fgDrf38/XFXBqDOWFh/xNzMTpW0yd0XZT+/UNKN7v7eYt/T2dnpDfVO2+5uvX71h3XSYGb0odebWnXj2mt174p3jTk0JaklJaWapKaUqcnSakmn1NKU0tyZU9S1ZokuPuvUiP8DAMTNzLa4e2eQY0Ob4bv7b8zs12b2Znd/WtLFkp4M63y1aN8f/5nm54W9JJ00mNH1D33zhMAflnRsWNlVEJc0OPq15187ps3PbVGTSW+Z264pTU0a8mGlLaUhH9aUprTapzRramtaJ7U26/3nzFPnGR1h/+cBSJiwd+lcJ6k7u0Nnj6Q/CPl8NWXe4VcqerycQZd+ub+v7HHdm5/Xm9rS+u2VcyVJfccG1T6lSW3N6dFj2lpSesvcmbpgSYc62lsnVA+AZAk18N19q6RALzUa0f7pszX/8InrFvunzw793K/2D+nbj+0LdOw7lnVoyZx2SSMXgpknteq1149Lci4KQA1J1N0yG838v/tCwRn+7Ws+FGNVJ3pwV68e3NVb8pj3nTNXp81sk+SaP2tkM9arRzNqaUqrOZ3S25fO1tJTpkVQLYBiCPw4XXWVTtLILH/e4Ve0f/ps3b7mQyfM72vBv299sewx558xSxcum603TW3VtClNOnJsUK8ezYx+Pr2tRSvmTefVAhCS0HbpTETD7dIpoOfZXl337cf14uGx71ErtEtncHhYhzPD8RQakpRJl6w8RauXztG0KWP7ES4IwIkq2aVD4Ne43r6Mujc9px9vf0mZoSFNaU4V3aXz+POv6pWjg+V/aIKlTfr9Cxaq8/Q3nfA1LghoRAQ+iup5tlffeex5He4fVEd7i6QTd+k8c+CoHtw1sZ1CceOCgEZD4GPSevsyevSZXj314iH1Hx8afTx/l86Dv3pZ2wNsA02StEk3rD1TZ86dpsP9A6OPczFArSLwEZndLx3RI7sPaGDIR7dq5u/S2fVSn+554kUl57esuEIXg+ltLZo3Y4qOHh/S/FltXBCQOAQ+EqW3L6Md+w/phYP9Y3bl5O/Sefo3R/SNR/cm9sLQmja5pCtXLdBFZ56qeTOmaP+hfknGKwPEisBHTcqNkfb29o1eFHJ6njuob27aq+Hk/LqOGv/KgFcFiBKBj7qUe6WQP3vPSeIFIfeq4MNvP0OXnTd/9BUBFwNUE4GPhlTugtC9ea9SKVNmMP7f+da0yc305+9ZrjPnThMXAkwUgQ8U0NuX0b6D/Zraktb+Q/2jF4YkXQy4EKBSBD5QoWIXg+ltLdr54mHd9qOdGorxTc2FLgQsFkMi8IGqyx8X5RZl7358n9Y/vCe2VwZpk7rWLNYFS2aPvgqY2pLm1UCDIfCBiBR6ZRDXq4KmlDQ4zKuBRkPgAwmQe1WQm8PndunsfPGwPvfjp2WmyF4ZpE3603cv1++sPJX3D9QZAh9IuPGvDOK4EHARqA8EPlCj4r4QcBGoPQQ+UGfGXwgefaZXX314jwbz1gjSJg2F8H/n/IsAC8LJQ+ADDWD8GsHR40PavKc31FcDrWmTzPQ/Vi8aszuIi0B8CHyggRV6NfC1nz0rM9OxgepvG8pdBK5911JeBcSAwAcwRpQXgfxtoasWd/DegJAR+ADKiuIikFtXGH8RIPyrh8AHMCFRXARaUpKlUoyAqoTAB1A1+ReB+7b/Rl/86a/G7A6aLLr/yUlM4JvZc5KOSBqSNFiuKAIfSL7xu4Pu2/4bffn+XZJMmUleCej+K5e0wO9091eCHE/gA7Up9ypg/LbQyb43gO6/PAIfQGzyR0DVfG8A3X9hSQr8ZyUdlOSS/sHd7yx1PIEP1Kfx6wDVGAHluv+bLz1LV606vXrF1pgkBf48d99vZidL+omk69z9oXHHdEnqkqSFCxe+be/evaHVAyAZio2AJuoj71isy86b35Bdf2ICf8yJzG6R1Ofunyt2DB0+0Hiq2f034sw/EYFvZlMlpdz9SPbjn0j6tLv/sNj3EPgAqtX958/8r1y1sG6DPymBv1jSd7OfNkn6lrv/danvIfAB5KtW99+cNn3it95cl11/IgJ/Igh8AKXkd/+f/fHTGqhwz2c9dv0EPoC619uX0bc2P9/wXT+BD6BhTHbm35KSlErV7PbOSgK/KexiACBMHe2t6mhv1VsXzNRlb5s/OvO/+xf79JUH9pT9/uPDkoaHddN3t2vfq6/X9fZOOnwAdat701596ns7Ku76a+lNXXT4ACDpqvNP19qVp1bc9WeGXJKPdv3XXLi4Lrp9OnwADSXX9UvS8YC7fJpT0jUXLk5k8LNoCwAlTHR7Z0rSn71neaK2dBL4ABDQRLZ3NqdNt6xbkYj5PoEPABWaSNf/yUvOVNeaJRFUVxyLtgBQofHbO9c/sqfsAu+tG3Zq/8F+XfSWU7Ri3vTEjHmKocMHgCK6N+3VX927XUMB3sSbNunT71sZ+ZiHDh8AqiC3rXPH/kPauPNlff3nxf9ex5BLN313u45mBmMf8xRD4ANACR3trVqz/GStWX6y5s1s060bdpY8Pvf1JIZ+Ku4CAKBWdK1Zor9+30o1p63kcbdu2Knb7ntKvX2ZiCoLhsAHgApcdf7p2vQXF+vj71muphIJ+pUH9+j8v/mpujcn58+2EvgAUKGO9lZdd/Eybf7ku3X1fyu+SDswNHJ7hjsfeibC6ooj8AFggjraW3XLupX65CVnljzu1g071b0p/k6fwAeASQoy2/+re7fHPtMn8AGgCnKz/Y+8c3HBrw8NS1/auCviqsYi8AGgSjraW3XD2rfo6gsKz/W//vO9sc7zCXwAqLLrLl5WNFxv3bAzttAn8AGgyjraW/WZ968s+vW4FnEJfAAIwVWrTi+5eyeORVwCHwBC0rVmSdHQHxqW1j9c/s8tVhOBDwAh6lqzpOgi7j88tCfSLp/AB4CQFVvEHXZpx/5DkdUReuCbWdrMfmFm3w/7XACQRB3trfqjdxTen//oM72R1RFFh/8nkp6K4DwAkFjXXLi4YOB+9eHoxjqhBr6ZzZf0XknrwzwPACRdsS5/cDi6sU7YHf7/kXS9pKJ/IMzMusysx8x6Dhw4EHI5ABCfFfOmF3z8cP9AJOcPLfDN7HclvezuW0od5+53ununu3fOmTMnrHIAoOGF2eGvlrTOzJ6T9B1JF5nZP4d4PgBACaEFvrv/hbvPd/dFkq6QtNHdPxjW+QAg6aa3tVT0eLWxDx8AIjJvxpSKHq+2SALf3R9w99+N4lwAkFT7D/VX9Hi10eEDQESK7cap+V06AICxtu8/XPBxZvgAUEd6+zIF746ZtuL786uNwAeACKx/eI8GC7wFtWvNYnW0t0ZSA4EPACEr1t1L0gVLZkdWB4EPACHbd7BfZnbC41GOcyQCHwBCt3lPr44P+QmP37D2zMjGORKBDwCh6u3L6LM/fvqEx5tT0qrFHZHWQuADQIjWP7xHAwW6e0ulNH9WW6S1EPgAEJLuTXv1lQcLL9befOlZkY5zJAIfAELR25fRLd/bUfBrH3nHYl21qvAfNg8TgQ8AIeje/HzBUU5z2nTNhYX/vm3YCHwAqLLdLx3RlzbuKvi1W9atiHyUk0PgA0AVdW/aq7X/9+GC3X1co5ycptjODAB1pnvTXt3079sLfq21Kb5RTg6BDwBVsPulI7q5yCJtS1NKn7387NhGOTkEPgBM0j1bX9An/u0JDRYY4zSlTRuue7uWnjIthsrGYoYPAJOw+6Uj+sRd2wreOkGSPrVuRSLCXqLDB4AJ6e3LqHvz8/rSxl0FF2ib0qZPXboi1kXa8Qh8AKjQPVtf0PV3PaHMYOGuviVt2vCxCxPT2ecQ+ABQgdERTrGwb0rpc5efnbiwlwh8AAgstzhbbF7fkqAF2kIIfAAIoNzibGt262VSw14i8AGgrFKdfXPa9LGLlunKVQtj32dfDoEPAEX09mW0Y/9hXV+ks0/q4mwxEw58M/sDd/+nEl+fIukhSa3Z89zl7jdP9HwAEKV7tr6gG+7eppRMmcHhE76e5MXZYibT4X9KUtHAl5SRdJG795lZs6RHzOw+d980iXMCQKjyu/pCQS8lf3G2mJKBb2bbin1J0imlvtfdXVJf9tPm7L/Cqx0AEKPevoz2HezX9hcO6TM/eLJoV39SS1rD7rr9strq7HPKdfinSPptSQfHPW6Sfl7uh5tZWtIWSUsl3eHumydSJACEJTe6SZvp6PGhose1Npn+/oPnacW8GYlfnC2mXOB/X1K7u28d/wUze6DcD3f3IUnnmNlMSd81s5XuPubeoWbWJalLkhYuXBi0bgCYlCCjG2lsV79m+ckRVlh9NjJ5ieBEZjdLOurunyt2TGdnp/f09ERSD4DGlb8g+/pA6a7+qx/qTHRXb2Zb3L0zyLGhbcs0szmSBtz9NTNrk/RuSbeFdT4AKCdoVz+1Na2h4fro6vOFuQ9/rqRvZOf4KUn/6u7fD/F8AFBUuW2WudHNX773LK08bYbmz2pLbFc/UaEFvrtvk3RuWD8fAILq7cvohru36dhA4a6+HhZkg+CdtgDqVm675aH+42pOpXRMYwO/nhZkgyDwAdSl3AinOZXS8aEhDY/bn9IoXX0+Ah9A3ch19FNb0qMjnFxX35QauaNlSzqlgeHhhunq8xH4AOpCfkefGRxSKmVjvt7W3KQ7rjpPM9qa63JBNggCH0DNKtXRa9zdLQeGh7Vi3vSGDPocAh9ATSrX0bemTW6m1rwRTiOHvUTgA6ghlXT0ljL94Nq36+jxoYYd4YxH4AOoCRPp6GvxjpZhIvABJFKum58/q02S6OirgMAHkDj53fzA8LA++s6lJ7xxio6+cgQ+gEQoNZ//8v27NPJnON5AR185Ah9A7MrN51vSaXWtWaw7Htg92vXT0VeOwAcQq/wbm5XaQ3/lqoW6ctXC0bk+HX3lCHwAkcpfjO1ob9W+g/2B5vO5gCfoJ47ABxCZ8Yuxt192tlYvna2B4bF3sWQ+H45U3AUAqF+9fRk98evX1NuXGTO6OZIZ1LGBYV1/9zZJ0u2Xna0pzSlNa23SlObU6Hz+rQtmEvZVRIcPIBRBtlY2p1Lad7Bf6845TauXzmY+HzICH0DVFVqILbS1cmB4ePSNVR3trQR9yBjpAJi0/NGNpNGF2Hwt6bSufdfSE0Y3hHx06PABTErQhVi2VsaPDh9AYOM7+UoXYnNjGxZj40GHDyCQQp386R1TWYitIXT4AMoq1slPbUkXHN3kL8TSzScHgQ/gBEEWYZtTKR09PlR0dIPkYaQDYIxKFmHnz2rTWxfMZHRTI0Lr8M1sgZndb2ZPmdkOM/uTsM4FoHLju/jcY5UuwkqMbmpFmB3+oKSPu/vjZjZN0hYz+4m7PxniOQEEUKiLX3fOaQVvZMYibP0ILfDd/UVJL2Y/PmJmT0k6TRKBD8So0Ltgr797m1Yvna35s9rKLsIS9LUrkkVbM1sk6VxJm6M4H4ARhcY2xRZg9x3sV0d7K4uwdSz0RVsza5d0t6Q/dffDBb7eJalLkhYuXBh2OUDDKDa2KdfFM7qpX6F2+GbWrJGw73b3/1foGHe/09073b1zzpw5YZYD1KVKFl97+zKBungWYetTaB2+mZmkf5T0lLt/PqzzAI1sIouvHe2tdPENKswOf7Wk35d0kZltzf67JMTzAQ2lVBdfbmwj0cU3otAC390fcXdz97Pd/Zzsvw1hnQ+oV4VGNhKLr6gc77QFEqzYyEYSi6+oGPfSARKq1MhGEouvqBgdPhCz3r5MwS683MKrRBePyhD4QIwmM7LJ4d2vCIqRDhCTaoxsgErQ4QMhY2SDpCDwgRAxskGSMNIBQsLIBklDhw9MQrFxjcTIBslD4AMTVGpcIzGyQfIw0gEmoNy4RmJkg+ShwweKmOy4RmJkg2Qh8IECqjWukRjZIDkY6QDjMK5BvaLDR8MpNaqRGNegfhH4aCjlRjUS4xrUL0Y6aBhBRjUS4xrULzp81I1qjWokxjWoTwQ+6kK1RzUS4xrUH0Y6qHmMaoBg6PCReIxqgOog8JFojGqA6mGkg8RiVANUFx0+EotRDVBdBD5iUW4uLzGqAaottJGOmX3NzF42s+1hnQO16Z6tL2j1bRv1wfWbtfq2jbp36wsFj2NUA1SXuXs4P9hsjaQ+Sd9095VBvqezs9N7enpCqQfJ0NuX0erbNurYwBud+5TmlH52w0VFgzzIqwGgUZnZFnfvDHJsaCMdd3/IzBaF9fORLEFDuZK5fA6jGqA6mOFj0oJsncypdC4PoHpi35ZpZl1m1mNmPQcOHIi7HFQo6NbJHObyQHxi7/Dd/U5Jd0ojM/yYy0GeIGOaiYxo2EIJxCP2wEcyBR3TTHREw1weiF6Y2zK/LelRSW82s31m9uGwzoXqqmRMw4gGqB1h7tL5QFg/G+GqdEzDiAaoDYx0GkQle9knMqZhRAMkH4HfACrZNim9Maa5ftz3EOhAbSPw61z+PD43orn+7m1avXR2yQBnTAPUHwK/BlUynpnItskcxjRAfSHwa0yl4xne2QogJ/Z32iK4St/VKrFtEsAb6PBryETHM8zjAUgEfuzC3i6ZwzweAIEfI7ZLAogSgR8TtksCiBqBHxO2SwKIGrt0qqS3L6Mnfv1ayR0z+dguCSBqdPhVUOksXmIeDyB6BP4kTXQWLzGPBxAtAn+SJjOLl5jHA4gOM/w8lc7hJWbxAGoHHX7WRObwErN4ALWDwNfk5vASs3gAtYHA1+Tn8BKzeADJV3czfObwAFBYXXX4zOEBoLi6CXzm8ABQWt0EPnN4ACitbmb4zOEBoLS6CXz+lB8AlFY3Ix2JOTwAlBJqh29ma83saTPbbWY3hnmunI72Vr11wUzCHgDGCS3wzSwt6Q5JvyPpLEkfMLOzwjofAKC0MDv8/yppt7vvcffjkr4j6fdCPB8AoIQwA/80Sb/O+3xf9rExzKzLzHrMrOfAgQMhlgMAjS3MwLcCj/kJD7jf6e6d7t45Z86cEMsBgMYWZuDvk7Qg7/P5kvaHeD4AQAnmfkLTXZ0fbNYk6VeSLpb0gqTHJF3p7jtKfM8BSXurVMJsSa9U6WdFoZbqpdbw1FK9tVSrVFv1VlLr6e4eaDwS2j58dx80s2sl/UhSWtLXSoV99nuqNtMxsx5376zWzwtbLdVLreGppXprqVaptuoNq9ZQ33jl7hskbQjzHACAYOrm1goAgNLqOfDvjLuACtVSvdQanlqqt5ZqlWqr3lBqDW3RFgCQLPXc4QMA8tR84Je7QZuZtZrZv2S/vtnMFkVf5Wgt5WpdY2aPm9mgmV0eR43j6ilX7/8ysyfNbJuZ/dTMTo+jzmwt5Wr9n2b2SzPbamaPxH1fp6A3FjSzy83MzSy23SUBnturzexA9rndambXxFFntpayz6uZ/ffs7+0OM/tW1DWOq6Xcc/uFvOf1V2b22qRO6O41+08j2z2fkbRYUoukJySdNe6YP5b099mPr5D0LwmudZGksyV9U9LlNfDcvkvSSdmPP5Lw53Z63sfrJP0wyc9t9rhpkh6StElSZ1JrlXS1pC/H9XxWWOsySb+QNCv7+clJrnfc8ddpZHv7hM9Z6x1+kBu0/Z6kb2Q/vkvSxWZW6LYPYStbq7s/5+7bJA0X+gERC1Lv/e7+evbTTRp5N3UcgtR6OO/TqSpwm48IBb2x4Gck3S7pWJTFjVNLN0EMUusfSrrD3Q9Kkru/HHGN+Sp9bj8g6duTOWGtB36QG7SNHuPug5IOSeqIpLoidWQVvJlcglRa74cl3RdqRcUFvVHfR83sGY2E6Mciqq2QsvWa2bmSFrj796MsrICgvweXZUd7d5nZggJfj0KQWpdLWm5mPzOzTWa2NrLqThT4/2PZcekZkjZO5oS1HvhBbtAW6CZuEUhKHUEFrtfMPiipU9JnQ62ouKA36rvD3ZdIukHS/w69quJK1mtmKUlfkPTxyCoqLshz+z1Ji9z9bEn/oTdeUUctSK1NGhnrvFMjHfN6M5sZcl3FVJIJV0i6y92HJnPCWg/8IDdoGz0me3+fGZJejaS6InVkJf1mcoHqNbN3S7pJ0jp3z0RU23iVPrffkfS+UCsqrVy90yStlPSAmT0n6XxJ98a0cFv2uXX33rz/7b8q6W0R1TZe0Dy4x90H3P1ZSU9r5AIQh0p+b6/QJMc5kmp+0bZJ0h6NvNTJLXqsGHfMRzV20fZfk1pr3rFfV/yLtkGe23M1sui0rAZqXZb38aWSepJc77jjH1B8i7ZBntu5eR+/X9KmBNe6VtI3sh/P1shIpSOp9WaPe7Ok55R939SkzhnHf2iVn7RLNHJXzmck3ZR97NMa6TglaYqkf5O0W9L/l7Q4wbX+F41c9Y9K6pW0I+HP7X9IeknS1uy/exNc6xcl7cjWeX+pgE1CveOOjS3wAz63f5N9bp/IPrdnJrhWk/R5SU9K+qWkK5L+eyDpFkl/W43z8U5bAGgQtT7DBwAEROADQIMg8AGgQRD4ANAgCHwAaBAEPlBC0LtaArWAbZlAEWaW1sge6fdo5P0Rj0n6gLs/GWthwATR4QPF1dKdIoGyCHyguFq7wylQEoEPFFdrdzgFSiLwgeJq7Q6nQEkEPlDcY5KWmdkZZtaikbut3htzTcCENcVdAJBU7j5oZtdK+pFG/v7o19x9R8xlARPGtkwAaBCMdACgQRD4ANAgCHwAaBAEPgA0CAIfABoEgQ8ADYLAB4AGQeADQIP4T3ZaNTBs8AGBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a308ce9b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.DataFrame(np.array(history[\"W\"])[:,:,0]).plot.scatter(0, 1)\n",
    "plt.scatter(W_true[:1], W_true[1:], color = \"red\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.1])"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W_true[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}