III_01_Symmetric_matrices.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ This notebook is part of lecture 25 *Symmetric matrices and positive definiteness* in the OCW MIT course 18.06 by Prof Gilbert Strang [1]\n",
    "+ Created by me, Dr Juan H Klopper\n",
    "    + Head of Acute Care Surgery\n",
    "    + Groote Schuur Hospital\n",
    "    + University Cape Town\n",
    "    + <a href=\"mailto:juan.klopper@uct.ac.za\">Email me with your thoughts, comments, suggestions and corrections</a> \n",
    "<a rel=\"license\" href=\"http://creativecommons.org/licenses/by-nc/4.0/\"><img alt=\"Creative Commons Licence\" style=\"border-width:0\" src=\"https://i.creativecommons.org/l/by-nc/4.0/88x31.png\" /></a><br /><span xmlns:dct=\"http://purl.org/dc/terms/\" href=\"http://purl.org/dc/dcmitype/InteractiveResource\" property=\"dct:title\" rel=\"dct:type\">Linear Algebra OCW MIT18.06</span> <span xmlns:cc=\"http://creativecommons.org/ns#\" property=\"cc:attributionName\">IPython notebook [2] study notes by Dr Juan H Klopper</span> is licensed under a <a rel=\"license\" href=\"http://creativecommons.org/licenses/by-nc/4.0/\">Creative Commons Attribution-NonCommercial 4.0 International License</a>.\n",
    "\n",
    "+ [1] <a href=\"http://ocw.mit.edu/courses/mathematics/18-06sc-linear-algebra-fall-2011/index.htm\">OCW MIT 18.06</a>\n",
    "+ [2] Fernando Pérez, Brian E. Granger, IPython: A System for Interactive Scientific Computing, Computing in Science and Engineering, vol. 9, no. 3, pp. 21-29, May/June 2007, doi:10.1109/MCSE.2007.53. URL: http://ipython.org"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>\n",
       "<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>\n",
       "<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>\n",
       "<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>\n",
       "<link href='http://fonts.googleapis.com/css?family=Philosopher:400,700,400italic,700italic' rel='stylesheet' type='text/css'>\n",
       "\n",
       "<style>\n",
       "\n",
       "@font-face {\n",
       "    font-family: \"Computer Modern\";\n",
       "    src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');\n",
       "}\n",
       "\n",
       "#notebook_panel { /* main background */\n",
       "    background: #ddd;\n",
       "    color: #000000;\n",
       "}\n",
       "\n",
       "\n",
       "\n",
       "/* Formatting for header cells */\n",
       ".text_cell_render h1 {\n",
       "    font-family: 'Philosopher', sans-serif;\n",
       "    font-weight: 400;\n",
       "    font-size: 2.2em;\n",
       "    line-height: 100%;\n",
       "    color: rgb(0, 80, 120);\n",
       "    margin-bottom: 0.1em;\n",
       "    margin-top: 0.1em;\n",
       "    display: block;\n",
       "}\t\n",
       ".text_cell_render h2 {\n",
       "    font-family: 'Philosopher', serif;\n",
       "    font-weight: 400;\n",
       "    font-size: 1.9em;\n",
       "    line-height: 100%;\n",
       "    color: rgb(200,100,0);\n",
       "    margin-bottom: 0.1em;\n",
       "    margin-top: 0.1em;\n",
       "    display: block;\n",
       "}\t\n",
       "\n",
       ".text_cell_render h3 {\n",
       "    font-family: 'Philosopher', serif;\n",
       "    margin-top:12px;\n",
       "    margin-bottom: 3px;\n",
       "    font-style: italic;\n",
       "    color: rgb(94,127,192);\n",
       "}\n",
       "\n",
       ".text_cell_render h4 {\n",
       "    font-family: 'Philosopher', serif;\n",
       "}\n",
       "\n",
       ".text_cell_render h5 {\n",
       "    font-family: 'Alegreya Sans', sans-serif;\n",
       "    font-weight: 300;\n",
       "    font-size: 16pt;\n",
       "    color: grey;\n",
       "    font-style: italic;\n",
       "    margin-bottom: .1em;\n",
       "    margin-top: 0.1em;\n",
       "    display: block;\n",
       "}\n",
       "\n",
       ".text_cell_render h6 {\n",
       "    font-family: 'PT Mono', sans-serif;\n",
       "    font-weight: 300;\n",
       "    font-size: 10pt;\n",
       "    color: grey;\n",
       "    margin-bottom: 1px;\n",
       "    margin-top: 1px;\n",
       "}\n",
       "\n",
       ".CodeMirror{\n",
       "        font-family: \"PT Mono\";\n",
       "        font-size: 100%;\n",
       "}\n",
       "\n",
       "</style>\n",
       "\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.core.display import HTML, Image\n",
    "css_file = 'style.css'\n",
    "HTML(open(css_file, 'r').read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<link href='http://fonts.googleapis.com/css?family=Alegreya+Sans:100,300,400,500,700,800,900,100italic,300italic,400italic,500italic,700italic,800italic,900italic' rel='stylesheet' type='text/css'>\n",
       "<link href='http://fonts.googleapis.com/css?family=Arvo:400,700,400italic' rel='stylesheet' type='text/css'>\n",
       "<link href='http://fonts.googleapis.com/css?family=PT+Mono' rel='stylesheet' type='text/css'>\n",
       "<link href='http://fonts.googleapis.com/css?family=Shadows+Into+Light' rel='stylesheet' type='text/css'>\n",
       "<link href='http://fonts.googleapis.com/css?family=Philosopher:400,700,400italic,700italic' rel='stylesheet' type='text/css'>\n",
       "\n",
       "<style>\n",
       "\n",
       "@font-face {\n",
       "    font-family: \"Computer Modern\";\n",
       "    src: url('http://mirrors.ctan.org/fonts/cm-unicode/fonts/otf/cmunss.otf');\n",
       "}\n",
       "\n",
       "#notebook_panel { /* main background */\n",
       "    background: #ddd;\n",
       "    color: #000000;\n",
       "}\n",
       "\n",
       "\n",
       "\n",
       "/* Formatting for header cells */\n",
       ".text_cell_render h1 {\n",
       "    font-family: 'Philosopher', sans-serif;\n",
       "    font-weight: 400;\n",
       "    font-size: 2.2em;\n",
       "    line-height: 100%;\n",
       "    color: rgb(0, 80, 120);\n",
       "    margin-bottom: 0.1em;\n",
       "    margin-top: 0.1em;\n",
       "    display: block;\n",
       "}\t\n",
       ".text_cell_render h2 {\n",
       "    font-family: 'Philosopher', serif;\n",
       "    font-weight: 400;\n",
       "    font-size: 1.9em;\n",
       "    line-height: 100%;\n",
       "    color: rgb(200,100,0);\n",
       "    margin-bottom: 0.1em;\n",
       "    margin-top: 0.1em;\n",
       "    display: block;\n",
       "}\t\n",
       "\n",
       ".text_cell_render h3 {\n",
       "    font-family: 'Philosopher', serif;\n",
       "    margin-top:12px;\n",
       "    margin-bottom: 3px;\n",
       "    font-style: italic;\n",
       "    color: rgb(94,127,192);\n",
       "}\n",
       "\n",
       ".text_cell_render h4 {\n",
       "    font-family: 'Philosopher', serif;\n",
       "}\n",
       "\n",
       ".text_cell_render h5 {\n",
       "    font-family: 'Alegreya Sans', sans-serif;\n",
       "    font-weight: 300;\n",
       "    font-size: 16pt;\n",
       "    color: grey;\n",
       "    font-style: italic;\n",
       "    margin-bottom: .1em;\n",
       "    margin-top: 0.1em;\n",
       "    display: block;\n",
       "}\n",
       "\n",
       ".text_cell_render h6 {\n",
       "    font-family: 'PT Mono', sans-serif;\n",
       "    font-weight: 300;\n",
       "    font-size: 10pt;\n",
       "    color: grey;\n",
       "    margin-bottom: 1px;\n",
       "    margin-top: 1px;\n",
       "}\n",
       "\n",
       ".CodeMirror{\n",
       "        font-family: \"PT Mono\";\n",
       "        font-size: 100%;\n",
       "}\n",
       "\n",
       "</style>\n",
       "\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.core.display import HTML, Image\n",
    "css_file = 'style.css'\n",
    "HTML(open(css_file, 'r').read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sympy import init_printing, Matrix, symbols, sqrt\n",
    "from warnings import filterwarnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init_printing(use_latex = 'mathjax')\n",
    "filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Symmetric matrices\n",
    "# Positive definite matrices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Symmetric matrices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Symmetric matrices are square with the following property\n",
    "$$ {A}={A}^{T} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* We are concerned with the eigenvalues and eigenvectors of symmetric matrices\n",
    "    * The eigenvalues are real\n",
    "    * The eigenvectors are orthogonal, or at least, can be chosen orthogonal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Considering proof of the real nature of eigenvalues we have the following\n",
    "    * Any matrix equation of the following example can be changed to its complex conjugate form by changing each element into its complex conjugate form (here marked with a bar over the top)\n",
    "    $$ {A}\\mathbf{x}={\\lambda}\\mathbf{x} \\\\ {A}\\overline{x}=\\overline{\\lambda}\\overline{x} $$\n",
    "    * We can take the complex conjugate transpose of **x** on both sides\n",
    "    $$ \\overline{x}^{T}A\\mathbf{x}={\\lambda}\\overline{x}^{T}\\mathbf{x}\\dots\\left(1\\right) $$\n",
    "    * In the complex conjugate form this becomes the following\n",
    "    $$ \\overline{x}^{T}{A}^{T}\\mathbf{x}=\\overline{x}^{T}{\\lambda}\\mathbf{x} $$\n",
    "    * Now if A is symmetric we use the fact that A=A<sup>T</sup>\n",
    "    $$ \\overline{x}^{T}{A}\\mathbf{x}=\\overline{x}^{T}\\overline{\\lambda}\\mathbf{x}\\dots\\left(2\\right) $$\n",
    "    * Note how the right-hand sides of (1) and (2) are equal and we therefor have the following\n",
    "    $$ \\lambda\\overline{x}^{T}\\mathbf{x}=\\overline\\lambda\\overline{x}^{T}\\mathbf{x} $$\n",
    "    * This means the following\n",
    "    $$ \\lambda=\\overline\\lambda $$\n",
    "    * The only ways that this is possible is if the imaginary part is zero and only real eigenvalues are possible\n",
    "    * Note also what happens if the complex conjugate of the vector **x** is multiplied by the vector itself\n",
    "        * Remember that **x**<sup>T</sup>**x** is a form of the dot product (which is the length squared)\n",
    "        * Any number times its complex conjugate gets rid of the imaginary part"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Consider the following symmetric matrix A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}5 & 2\\\\2 & 3\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡5  2⎤\n",
       "⎢    ⎥\n",
       "⎣2  3⎦"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = Matrix([[5, 2], [2, 3]])\n",
    "A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Let's see if it really is symmetric by making sure that it is equal to it's transpose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A == A.transpose() # Boolean (true or false) statement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "S, D = A.diagonalize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* S, the matrix containing the eigenvectors as it's columns\n",
    "    * Remember that these eigenvectors are not necessarily the same as those you would get doing these problems by hand\n",
    "    * When substituting the values for &#955;<sub>i</sub> a singular matrix is created with rows that are simply linear combinations of each other\n",
    "    * You are free to choose values for the components of the eigenvectors for each eigenvalue (usually choosing the simplest ones)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}- \\frac{2}{1 + \\sqrt{5}} & - \\frac{2}{- \\sqrt{5} + 1}\\\\1 & 1\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡   -2          -2     ⎤\n",
       "⎢─────────  ───────────⎥\n",
       "⎢      ___      ___    ⎥\n",
       "⎢1 + ╲╱ 5   - ╲╱ 5  + 1⎥\n",
       "⎢                      ⎥\n",
       "⎣    1           1     ⎦"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* D, the matrix containing the values of the eigenvalues down the main diagonal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}- \\sqrt{5} + 4 & 0\\\\0 & \\sqrt{5} + 4\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡    ___               ⎤\n",
       "⎢- ╲╱ 5  + 4      0    ⎥\n",
       "⎢                      ⎥\n",
       "⎢               ___    ⎥\n",
       "⎣     0       ╲╱ 5  + 4⎦"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* In decomposition, a symmetric matrix results in the following\n",
    "$$ {A}={S}{\\Lambda}{S}^{T} $$\n",
    "* In this case we have an orthogonal matrix times diagonal matrix times transpose of orthogonal matrix\n",
    "$$ {A}={Q}{\\Lambda}{Q}^{T} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\begin{Bmatrix}- \\sqrt{5} + 4 : 1, & \\sqrt{5} + 4 : 1\\end{Bmatrix}$$"
      ],
      "text/plain": [
       "⎧    ___           ___       ⎫\n",
       "⎨- ╲╱ 5  + 4: 1, ╲╱ 5  + 4: 1⎬\n",
       "⎩                            ⎭"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.eigenvals()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\begin{bmatrix}\\begin{pmatrix}- \\sqrt{5} + 4, & 1, & \\begin{bmatrix}\\left[\\begin{matrix}- \\frac{2}{1 + \\sqrt{5}}\\\\1\\end{matrix}\\right]\\end{bmatrix}\\end{pmatrix}, & \\begin{pmatrix}\\sqrt{5} + 4, & 1, & \\begin{bmatrix}\\left[\\begin{matrix}- \\frac{2}{- \\sqrt{5} + 1}\\\\1\\end{matrix}\\right]\\end{bmatrix}\\end{pmatrix}\\end{bmatrix}$$"
      ],
      "text/plain": [
       "⎡⎛    ___         ⎡⎡   -2    ⎤⎤⎞  ⎛  ___         ⎡⎡    -2     ⎤⎤⎞⎤\n",
       "⎢⎜- ╲╱ 5  + 4, 1, ⎢⎢─────────⎥⎥⎟, ⎜╲╱ 5  + 4, 1, ⎢⎢───────────⎥⎥⎟⎥\n",
       "⎢⎜                ⎢⎢      ___⎥⎥⎟  ⎜              ⎢⎢    ___    ⎥⎥⎟⎥\n",
       "⎢⎜                ⎢⎢1 + ╲╱ 5 ⎥⎥⎟  ⎜              ⎢⎢- ╲╱ 5  + 1⎥⎥⎟⎥\n",
       "⎢⎜                ⎢⎢         ⎥⎥⎟  ⎜              ⎢⎢           ⎥⎥⎟⎥\n",
       "⎣⎝                ⎣⎣    1    ⎦⎦⎠  ⎝              ⎣⎣     1     ⎦⎦⎠⎦"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.eigenvects()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* We've seen in our example that, indeed, the eigenvalues are real\n",
    "* Let's see of the eigenvectors are orthogonal by looking at their dot product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\frac{4}{\\left(1 + \\sqrt{5}\\right) \\left(- \\sqrt{5} + 1\\right)} + 1$$"
      ],
      "text/plain": [
       "            4                \n",
       "───────────────────────── + 1\n",
       "⎛      ___⎞ ⎛    ___    ⎞    \n",
       "⎝1 + ╲╱ 5 ⎠⋅⎝- ╲╱ 5  + 1⎠    "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eigenvec_1 = Matrix([-2 / (1 + sqrt(5)), 1])\n",
    "eigenvec_2 = Matrix([-2 / (1 - sqrt(5)), 1])\n",
    "eigenvec_1.dot(eigenvec_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* This is certainly zero when simplified"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$0$$"
      ],
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(eigenvec_1.dot(eigenvec_2)).simplify() # Using the simplify() method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* We need not use symbolic computing (computer algebra system, CAS)\n",
    "* Let's look at numerical evaluation using numerical python (numpy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np # Using namespace abbreviations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[5, 2],\n",
       "        [2, 3]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = np.matrix([[5, 2], [2, 3]])\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w, v = np.linalg.eig(A) # Calculating the eigenvalues and eigenvectors\n",
    "# The result of np.linalg.eig() is a tuple, the first being the eigenvalues\n",
    "# The second being the eigenvectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 6.23606798,  1.76393202])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[ 0.85065081, -0.52573111],\n",
       "        [ 0.52573111,  0.85065081]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[ 6.23606798,  0.        ],\n",
       "        [ 0.        ,  1.76393202]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating the diagonal matrix manually from the eigenvalues\n",
    "D = np.matrix([[6.23606798, 0], [0, 1.76393202]])\n",
    "D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[ 5.,  2.],\n",
       "        [ 2.,  3.]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking to see if our equation for A holds\n",
    "v * D * np.matrix.transpose(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Positive definite matrices (referring to symmetric matrices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* The properties of positive definite (symmetric) matrices \n",
    "    * All eigenvalues are positive\n",
    "    * All pivots are positive\n",
    "    * All determinants (actually also all *sub*-determinants) are positive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "* The fact that a (square symmetric) matrix A is invertible implies the following\n",
    "    * The determinant is non-zero (actually larger than zero)\n",
    "    * The determinant is the product of the eigenvalues\n",
    "    * The determinant must therefor be larger than zero"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* For projection matrices\n",
    "    * The eigenvalues are either 0 or 1\n",
    "    * If this projection matrix is positive definite\n",
    "        * The eigenvalues must all be 1 (since they must be larger than zero)\n",
    "    * The only matrix that satisfies this property is the identity matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* The diagonal matrix D is positive definite\n",
    "    * This means that for any non-zero vector **x** we have **x**<sup>T</sup>D**x**>0\n",
    "    * Let's look at a 3-component vector with a 3&#215;3 matrix D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d1, d2, d3, x1, x2, x3 = symbols('d1 d2 d3 x1 x2 x3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\begin{pmatrix}\\left[\\begin{matrix}x_{1} & x_{2} & x_{3}\\end{matrix}\\right], & \\left[\\begin{matrix}d_{1} & 0 & 0\\\\0 & d_{2} & 0\\\\0 & 0 & d_{3}\\end{matrix}\\right], & \\left[\\begin{matrix}x_{1}\\\\x_{2}\\\\x_{3}\\end{matrix}\\right]\\end{pmatrix}$$"
      ],
      "text/plain": [
       "⎛[x₁  x₂  x₃], ⎡d₁  0   0 ⎤, ⎡x₁⎤⎞\n",
       "⎜              ⎢          ⎥  ⎢  ⎥⎟\n",
       "⎜              ⎢0   d₂  0 ⎥  ⎢x₂⎥⎟\n",
       "⎜              ⎢          ⎥  ⎢  ⎥⎟\n",
       "⎝              ⎣0   0   d₃⎦  ⎣x₃⎦⎠"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D = Matrix([[d1, 0, 0], [0, d2, 0], [0, 0, d3]])\n",
    "x_vect = Matrix([x1, x2, x3])\n",
    "x_vect.transpose(), D, x_vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Indeed we have **x**<sup>T</sup>D**x**>0 since the components if **x** are squared and the eigenvalues are all positive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}d_{1} x_{1}^{2} + d_{2} x_{2}^{2} + d_{3} x_{3}^{2}\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡     2        2        2⎤\n",
       "⎣d₁⋅x₁  + d₂⋅x₂  + d₃⋅x₃ ⎦"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_vect.transpose() * D * x_vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Not all symmetric matrices with a positive determinant are definite positive\n",
    "* Easy matrices to construct with this property have negative values on the main diagonal\n",
    "* Note below how the eigenvalues are not all more than zero\n",
    "* Also note how **x**<sup>T</sup>D**x**&#8815;0\n",
    "* It is important to note that the *sub*-determinant must also be positive\n",
    "    * In the example below the *sub*-determinant of 3 is -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}3 & 1\\\\1 & -1\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡3  1 ⎤\n",
       "⎢     ⎥\n",
       "⎣1  -1⎦"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = Matrix([[3, 1], [1, -1]])\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A == A.transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$-4$$"
      ],
      "text/plain": [
       "-4"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.det()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\begin{Bmatrix}1 + \\sqrt{5} : 1, & - \\sqrt{5} + 1 : 1\\end{Bmatrix}$$"
      ],
      "text/plain": [
       "⎧      ___         ___       ⎫\n",
       "⎨1 + ╲╱ 5 : 1, - ╲╱ 5  + 1: 1⎬\n",
       "⎩                            ⎭"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.eigenvals()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\begin{bmatrix}\\begin{pmatrix}1 + \\sqrt{5}, & 1, & \\begin{bmatrix}\\left[\\begin{matrix}- \\frac{1}{- \\sqrt{5} + 2}\\\\1\\end{matrix}\\right]\\end{bmatrix}\\end{pmatrix}, & \\begin{pmatrix}- \\sqrt{5} + 1, & 1, & \\begin{bmatrix}\\left[\\begin{matrix}- \\frac{1}{2 + \\sqrt{5}}\\\\1\\end{matrix}\\right]\\end{bmatrix}\\end{pmatrix}\\end{bmatrix}$$"
      ],
      "text/plain": [
       "⎡⎛      ___     ⎡⎡    -1     ⎤⎤⎞  ⎛    ___         ⎡⎡   -1    ⎤⎤⎞⎤\n",
       "⎢⎜1 + ╲╱ 5 , 1, ⎢⎢───────────⎥⎥⎟, ⎜- ╲╱ 5  + 1, 1, ⎢⎢─────────⎥⎥⎟⎥\n",
       "⎢⎜              ⎢⎢    ___    ⎥⎥⎟  ⎜                ⎢⎢      ___⎥⎥⎟⎥\n",
       "⎢⎜              ⎢⎢- ╲╱ 5  + 2⎥⎥⎟  ⎜                ⎢⎢2 + ╲╱ 5 ⎥⎥⎟⎥\n",
       "⎢⎜              ⎢⎢           ⎥⎥⎟  ⎜                ⎢⎢         ⎥⎥⎟⎥\n",
       "⎣⎝              ⎣⎣     1     ⎦⎦⎠  ⎝                ⎣⎣    1    ⎦⎦⎠⎦"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.eigenvects()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "S, D = A.diagonalize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}- \\frac{1}{- \\sqrt{5} + 2} & - \\frac{1}{2 + \\sqrt{5}}\\\\1 & 1\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡    -1          -1    ⎤\n",
       "⎢───────────  ─────────⎥\n",
       "⎢    ___            ___⎥\n",
       "⎢- ╲╱ 5  + 2  2 + ╲╱ 5 ⎥\n",
       "⎢                      ⎥\n",
       "⎣     1           1    ⎦"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}1 + \\sqrt{5} & 0\\\\0 & - \\sqrt{5} + 1\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡      ___             ⎤\n",
       "⎢1 + ╲╱ 5        0     ⎥\n",
       "⎢                      ⎥\n",
       "⎢               ___    ⎥\n",
       "⎣    0      - ╲╱ 5  + 1⎦"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}x_{1}\\\\x_{2}\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡x₁⎤\n",
       "⎢  ⎥\n",
       "⎣x₂⎦"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_vect = Matrix([x1, x2])\n",
    "x_vect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}x_{1}^{2} \\left(1 + \\sqrt{5}\\right) + x_{2}^{2} \\left(- \\sqrt{5} + 1\\right)\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡  2 ⎛      ___⎞     2 ⎛    ___    ⎞⎤\n",
       "⎣x₁ ⋅⎝1 + ╲╱ 5 ⎠ + x₂ ⋅⎝- ╲╱ 5  + 1⎠⎦"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_vect.transpose() * D * x_vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* In this example the *sub*-determinant of 1 is -3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}-3 & 1\\\\1 & 1\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡-3  1⎤\n",
       "⎢     ⎥\n",
       "⎣1   1⎦"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = Matrix([[-3, 1], [1, 1]])\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A == A.transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "S, D = A.diagonalize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$$\\left[\\begin{matrix}x_{1}^{2} \\left(-1 + \\sqrt{5}\\right) + x_{2}^{2} \\left(- \\sqrt{5} - 1\\right)\\end{matrix}\\right]$$"
      ],
      "text/plain": [
       "⎡  2 ⎛       ___⎞     2 ⎛    ___    ⎞⎤\n",
       "⎣x₁ ⋅⎝-1 + ╲╱ 5 ⎠ + x₂ ⋅⎝- ╲╱ 5  - 1⎠⎦"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_vect.transpose() * D * x_vect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}