HW5.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HW5.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Q1\n"
      ],
      "metadata": {
        "id": "hBxIF2gtZuJD"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ROwMak6DZsBf",
        "outputId": "33b77f32-b5d9-4620-c00e-1f8f3fb54884"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Policy {-1: 0, 1: 1, 2: 1}\n",
            "A value -1.1666666666666667\n",
            "b value 1.0\n",
            "w value -0.8571428571428571\n",
            "Calculated Q(s,a) {-1: [0.8571428571428571, -0.0, 0.8571428571428571], 1: [0.8571428571428571, -0.0, -2.571428571428571], 2: [0.8571428571428571, -0.0, -4.285714285714286]}\n",
            "Policy {-1: -1, 1: -1, 2: -1}\n",
            "A value 3.933333333333333\n",
            "b value 1.0\n",
            "w value 0.2542372881355932\n",
            "Calculated Q(s,a) {-1: [-0.2542372881355932, 0.0, -0.2542372881355932], 1: [-0.2542372881355932, 0.0, 0.7627118644067796], 2: [-0.2542372881355932, 0.0, 1.271186440677966]}\n",
            "Policy {-1: 0, 1: 1, 2: 1}\n"
          ]
        }
      ],
      "source": [
        "D = [(1,1,1,2),(2,0,-1,1),(1,-1,0,-1)]\n",
        "ActionList = [-1,0,1]\n",
        "gamma = 0.9\n",
        "policy = {-1:0,1:1,2:1}\n",
        "wprev = 1\n",
        "print(\"Initial Policy\",policy)\n",
        "for iteration in range(2):\n",
        "    A = 0\n",
        "    b = 0\n",
        "    for traj in D:\n",
        "        si,ai,rnext,snext = traj\n",
        "        anext = policy[snext]\n",
        "        A += (si*(ai**2) + ai*si + ai) * ( (si*(ai**2) + ai*si + ai) - gamma*(snext*(anext**2) + anext*snext + anext) )\n",
        "        b += rnext * (ai**2 + ai*si + ai)\n",
        "\n",
        "    A = A/len(D)\n",
        "    b = b/len(D)\n",
        "    wnext = (1/A)*b\n",
        "    Q = {-1:[wnext*(-1*(x**2)+(-1*x)+x) for x in ActionList],1:[wnext*(1*(x**2)+(1*x)+x) for x in ActionList],2:[wnext*(2*(x**2)+(2*x)+x) for x in ActionList]}\n",
        "    for k in Q.keys():\n",
        "        policy[k] = ActionList[Q[k].index(max(Q[k]))]\n",
        "    print(\"A value\",A)\n",
        "    print(\"b value\",b)\n",
        "    print(\"w value\",wnext)\n",
        "    print(\"Calculated Q(s,a)\",Q)\n",
        "    print(\"Policy\",policy)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Q2\n"
      ],
      "metadata": {
        "id": "umCLUpFShojX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from numpy.linalg import inv\n",
        "D = [(1,1,1,2),(2,0,-1,1),(1,-1,0,-1)]\n",
        "ActionList = [-1,0,1]\n",
        "gamma = 0.9\n",
        "policy = {-1:0,1:1,2:1}\n",
        "w0 = np.reshape(np.array([1,1]),(2,1))\n",
        "Q = {-1:[float(np.matmul(np.reshape(np.array([-1*x+x,-1*(x**2)]),(1,2)),w0)) for x in ActionList],\\\n",
        "    1:[float(np.matmul(np.reshape(np.array([x+x,(x**2)]),(1,2)),w0)) for x in ActionList],\\\n",
        "    2:[float(np.matmul(np.reshape(np.array([2*x,2*(x**2)]),(1,2)),w0)) for x in ActionList]}\n",
        "\n",
        "for k in Q.keys():\n",
        "    policy[k] = ActionList[Q[k].index(max(Q[k]))]\n",
        "print(\"Initial Policy\",policy)\n",
        "\n",
        "for iteration in range(2):\n",
        "    A = np.zeros((2,2))\n",
        "    b = np.zeros((2,1))\n",
        "    for traj in D:\n",
        "        si,ai,rnext,snext = traj\n",
        "        anext = policy[snext]\n",
        "        phi_now = np.reshape(np.array([ai*si+ai,si*(ai**2)]),(2,1))\n",
        "        phi_next = np.reshape(np.array([anext*snext+anext,snext*(anext**2)]),(2,1)) \n",
        "        val = phi_now + gamma*phi_next\n",
        "        val = val.T\n",
        "        A += np.matmul(phi_now,val)\n",
        "        b += rnext*phi_now\n",
        "\n",
        "    A = A/len(D)\n",
        "    b = b/len(D)\n",
        "    Ainv = inv(A)\n",
        "    wnext = np.matmul(Ainv,b)\n",
        "    Q = {-1:[float(np.matmul(np.reshape(np.array([-1*x+x,-1*(x**2)]),(1,2)),wnext)) for x in ActionList],\\\n",
        "        1:[float(np.matmul(np.reshape(np.array([x+x,(x**2)]),(1,2)),wnext)) for x in ActionList],\\\n",
        "        2:[float(np.matmul(np.reshape(np.array([2*x,2*(x**2)]),(1,2)),wnext)) for x in ActionList]}\n",
        "    for k in Q.keys():\n",
        "        policy[k] = ActionList[Q[k].index(max(Q[k]))]\n",
        "    print(\"A value\",A)\n",
        "    print(\"b shape\",b)\n",
        "    print(\"w\",wnext)\n",
        "    print(\"Calculated Q(s,a)\",Q)\n",
        "    print(\"Policy\",policy)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ov6FV9zNbou_",
        "outputId": "453ad708-e877-4c93-8acb-da242adab872"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Policy {-1: 0, 1: 1, 2: 1}\n",
            "A value [[ 0.86666667  1.8       ]\n",
            " [-0.9         0.96666667]]\n",
            "b shape [[0.66666667]\n",
            " [0.33333333]]\n",
            "w [[0.01808318]\n",
            " [0.36166365]]\n",
            "Calculated Q(s,a) {-1: [-0.36166365280289325, 0.0, -0.36166365280289325], 1: [0.325497287522604, 0.0, 0.3978300180831825], 2: [0.6871609403254972, 0.0, 0.7594936708860758]}\n",
            "Policy {-1: 0, 1: 1, 2: 1}\n",
            "A value [[4.46666667 1.2       ]\n",
            " [0.9        1.26666667]]\n",
            "b shape [[0.66666667]\n",
            " [0.33333333]]\n",
            "w [[0.09708738]\n",
            " [0.19417476]]\n",
            "Calculated Q(s,a) {-1: [-0.19417475728155337, 0.0, -0.19417475728155337], 1: [-1.1102230246251565e-16, 0.0, 0.38834951456310685], 2: [0.19417475728155326, 0.0, 0.5825242718446602]}\n",
            "Policy {-1: 0, 1: 1, 2: 1}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "bEwEy6Hex_ng"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}