CSS 420 Lecture 2 Optimisation basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lecture 2\n",
    "===========================================================================\n",
    "\n",
    "Visualising objective functions\n",
    "------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sympy import *\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from matplotlib import cm\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We start with a simple quadratic - unimodal with minimum at 0,0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.gca(projection='3d')\n",
    "x1 = np.linspace(-2.0, 2.0, 20)\n",
    "y1 = np.linspace(-2.0, 2.0, 20)\n",
    "x1, y1 = np.meshgrid(x1, y1)\n",
    "z1 = x1**2 + y1**2\n",
    "surf = ax.plot_surface(x1, y1, z1, rstride=1, cstride=1, cmap=cm.jet,\n",
    "        linewidth=0, antialiased=False)\n",
    "fig.colorbar(surf, shrink=0.5, aspect=5)\n",
    "plt.show()\n",
    "plt.contourf(x1, y1, z1, cmap='binary_r')\n",
    "plt.scatter(0, 0, s=100, color='red')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Multimodal function similar to the one in the notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.gca(projection='3d')\n",
    "x2 = np.linspace(-1.0, 1.0, 20)\n",
    "y2 = np.linspace(-1.0, 1.0, 20)\n",
    "x2, y2 = np.meshgrid(x2, y2)\n",
    "z2 = x2**4 - x2**2 - 0.1 * x2 + y2**2\n",
    "surf = ax.plot_surface(x2, y2, z2, rstride=1, cstride=1, cmap=cm.jet,\n",
    "        linewidth=0, antialiased=False)\n",
    "fig.colorbar(surf, shrink=0.5, aspect=5)\n",
    "plt.show()\n",
    "plt.contourf(x2, y2, z2, cmap='binary_r')\n",
    "plt.scatter(0.7, 0, s=100, color='red')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now a simple function showing a saddle point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.gca(projection='3d')\n",
    "z22 = x2**2 - y2**2\n",
    "surf = ax.plot_surface(x2, y2, z22, rstride=1, cstride=1, cmap=cm.jet,\n",
    "        linewidth=0, antialiased=False)\n",
    "fig.colorbar(surf, shrink=0.5, aspect=5)\n",
    "plt.show()\n",
    "plt.contourf(x2, y2, z22, cmap='binary_r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Matrix Algebra\n",
    "------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "init_printing(use_latex='mathjax')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A = Matrix([[2, 5, 7], \n",
    "            [9, 6, 2],\n",
    "            [1, 3, 2]])\n",
    "B = Matrix([[2, 5, 7],\n",
    "            [4, 8, 6],\n",
    "            [1, 0, 1]])\n",
    "A, B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "(A == B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A + B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A * B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vectors are created in the same way"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "v = Matrix([1, 2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A*v"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remember the inner dimensions must agree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "v.T*A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = var('y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A = Matrix([[y, 4, 2], \n",
    "            [2, 1, 3], \n",
    "            [1, 0, 2]])\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A.det()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "A.inv()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's a bit difficult to work with, lets ask SymPy to simplify it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "simplify(A.inv())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "I = eye(3)\n",
    "I"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "C = Matrix([[4, -5], [2, -3]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "C.eigenvals()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "C.eigenvects()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Work Session 2\n",
    "------------\n",
    "Question 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Question 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = np.linspace(-1.0, 1.0, 20)\n",
    "plt.plot(x, 6*x-2, label='f1')\n",
    "plt.plot(x, 4*x**2-1, 'r', label='f2')\n",
    "plt.legend(loc='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.gca(projection='3d')\n",
    "x3 = np.linspace(-40.0, 40.0, 50)\n",
    "y3 = np.linspace(-40.0, 40.0, 50)\n",
    "x3, y3 = np.meshgrid(x3, y3)\n",
    "z3 = - np.exp(1)**(-(x3**2 + y3**2))\n",
    "surf = ax.plot_surface(x3, y3, z3, rstride=1, cstride=1, cmap=cm.jet,\n",
    "        linewidth=0, antialiased=False)\n",
    "fig.colorbar(surf, shrink=0.5, aspect=5)\n",
    "plt.show()\n",
    "plt.contourf(x3, y3, z3, cmap='binary_r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.gca(projection='3d')\n",
    "x4 = np.linspace(-100.0, 100.0, 50)\n",
    "y4 = np.linspace(-100.0, 100.0, 50)\n",
    "x4, y4 = np.meshgrid(x4, y4)\n",
    "z4 = x4**3 - 4.0 * x4**2 * y4 + x4 * y4**2\n",
    "surf = ax.plot_surface(x4, y4, z4, rstride=1, cstride=1, cmap=cm.jet,\n",
    "        linewidth=0, antialiased=False)\n",
    "fig.colorbar(surf, shrink=0.5, aspect=5)\n",
    "plt.show()\n",
    "plt.contourf(x4, y4, z4, cmap='binary_r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Question 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x, y = var('x y')\n",
    "f = x**2 + x * y + y**2\n",
    "dfx = diff(f,x)\n",
    "dfx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dfy = diff(f,y)\n",
    "dfy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "solve([dfx, dfy], [x, y])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "z5 = x2**2 + x2 * y2 + y2**2\n",
    "plt.contourf(x2, y2, z5, cmap='binary_r')\n",
    "plt.scatter(0, 0, s=10, color='red')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The contour plot along with a 3D plot will give us a pretty good idea where the minimum is:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib qt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.gca(projection='3d')\n",
    "surf = ax.plot_surface(x2, y2, z5, rstride=1, cstride=1, cmap=cm.jet,\n",
    "        linewidth=0, antialiased=False)\n",
    "fig.colorbar(surf, shrink=0.5, aspect=5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Question 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "D = Matrix([[2, 4, 5],\n",
    "            [6, 1, 9],\n",
    "            [3, 7, 8]])\n",
    "EV = D.eigenvals()\n",
    "EV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is excessive, lets enumerate them:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "EVs = [N(i,4) for i in EV]\n",
    "EVs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}