QLearning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.random import rand\n",
    "import math\n",
    "from random import randint\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import random\n",
    "from copy import deepcopy \n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class ConnectN:\n",
    "    \"\"\"\n",
    "    Connect N game simulator for two players, 1 and -1.\n",
    "    \n",
    "    Inputs:\n",
    "    Grid size- creates a grid size x grid size square board\n",
    "    N- number of tokens a player must connect to win the game\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, grid_size, n):\n",
    "        self.n = n\n",
    "        self.grid_size = grid_size\n",
    "        self.grid = np.zeros([grid_size,grid_size])\n",
    "        self.finished = 0\n",
    "        self.turn_num = 0\n",
    "        \n",
    "    def reset(self):\n",
    "        self.__init__(self.grid_size, self.n)\n",
    "\n",
    "    def check_win(self, col, row, player):\n",
    "        \"\"\"\n",
    "        Checks if given player has connected N tokens.\n",
    "        \"\"\"\n",
    "        for i in range(0, self.n):\n",
    "            if sum(self.grid[col, row - i:row - i + self.n]) == self.n*player:\n",
    "                self.finished = 1\n",
    "                return 1\n",
    "            if sum(self.grid[col - i: col - i + self.n, row]) == self.n*player:\n",
    "                self.finished = 1\n",
    "                return 1\n",
    "            if col - i >= 0 and col - i + self.n - 1 < self.grid_size and row - i >= 0 and row - i + self.n - 1 < self.grid_size:\n",
    "                if sum([self.grid[col - i + x, row - i + x] for x in range(0, self.n)]) == self.n*player:\n",
    "                    self.finished = 1\n",
    "                    return 1\n",
    "            if col - i >= 0 and col - i + self.n - 1 < self.grid_size and row + i >= self.n - 1 and row + i < self.grid_size:\n",
    "                if sum([self.grid[col - i + x, row + i - x] for x in range(0, self.n)]) == self.n*player:\n",
    "                    self.finished = 1\n",
    "                    return 1\n",
    "        return 0\n",
    "\n",
    "    def move(self, col, player):\n",
    "        \"\"\"\n",
    "        Given player and column to move in, modifies board and increments the turn counter.\n",
    "        \n",
    "        Returns a tuple, where first value is return message and second value is reward.\n",
    "        \"\"\"\n",
    "        self.turn_num += 1\n",
    "        \n",
    "        if self.finished == 1:\n",
    "            return 1, 50\n",
    "        sum_col = np.sum([abs(x) for x in self.grid[col]])\n",
    "        if sum_col == self.grid_size:\n",
    "            return -1, -1\n",
    "        self.grid[col, sum_col] = player\n",
    "        if self.check_win(col, sum_col, player) == 1:\n",
    "            return 1, 50\n",
    "        return 0, 0\n",
    "    \n",
    "    def simulate_move(self, col, player):\n",
    "        \"\"\"\n",
    "        Tests a move and returns if it is valid or not\n",
    "        \"\"\"\n",
    "        sum_col = np.sum([abs(x) for x in self.grid[col]])\n",
    "        if sum_col == self.grid_size:\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "        \n",
    "    def turn(self):\n",
    "        \"\"\"\n",
    "        Returns which player's turn it is. First turn is player 1, second turn is player -1.\n",
    "        \"\"\"\n",
    "        if self.turn_num%2 == 0:\n",
    "            return 1\n",
    "        else:\n",
    "            return -1\n",
    "        \n",
    "    def next_possible_moves(self):\n",
    "        \"\"\"\n",
    "        Returns array of possible columns for a next move\n",
    "        \"\"\"\n",
    "        columns = []\n",
    "        \n",
    "        for i in xrange(0, self.grid_size):\n",
    "            if (0 in self.grid[i]):\n",
    "                columns.append(i)\n",
    "                \n",
    "        return columns\n",
    "    \n",
    "    def all_tokens_placed(self):\n",
    "        \"\"\"\n",
    "        Returns location of all tokens (column, row) that have been placed\n",
    "        \"\"\"\n",
    "        all_tokens = []\n",
    "        \n",
    "        for col in xrange(0, self.grid_size):\n",
    "            for row in xrange(0, self.grid_size): \n",
    "                if self.grid[col][row] != 0:\n",
    "                    all_tokens.append({\"location\": [col, row], \"player\": self.grid[col][row]})\n",
    "                    \n",
    "        return all_tokens\n",
    "    \n",
    "    def is_empty(self, col, row):\n",
    "        \"\"\"\n",
    "        Returns if a given spot (column, row) is empty\n",
    "        \"\"\"\n",
    "        return self.grid[col][row] == 0\n",
    "    \n",
    "    \"\"\"\n",
    "    Following streak functions check if player has token streak in the four possible win directions\n",
    "    \"\"\"\n",
    "    def streakVertical(self, board, col, row, player):\n",
    "        if row > len(board[col]) - self.n:\n",
    "            return 0\n",
    "        for i in range(0,self.n):\n",
    "            if board[col][row + i] == -1*player:\n",
    "                return 0\n",
    "            if board[col][row + i] == 0:\n",
    "                return i\n",
    "        return self.n\n",
    "\n",
    "    def streakHorizontal(self, board, col, row, player):\n",
    "        if col > len(board) - self.n:\n",
    "            return 0\n",
    "        for i in range(0,self.n):\n",
    "            if board[col + i][row] == -1*player:\n",
    "                return 0\n",
    "            if board[col + i][row] == 0:\n",
    "                return i\n",
    "        return self.n\n",
    "    \n",
    "    def streakDiagonalUp(self, board, col, row, player):\n",
    "        if row > len(board[col]) - self.n or col > len(board) - self.n:\n",
    "            return 0\n",
    "        for i in range(0,self.n):\n",
    "            if board[col + i][row + i] == -1*player:\n",
    "                return 0\n",
    "            if board[col + i][row + i] == 0:\n",
    "                return i\n",
    "        return self.n\n",
    "    \n",
    "    def streakDiagonalDown(self, board, col, row, player):\n",
    "        if row < self.n or col > len(board) - self.n:\n",
    "            return 0\n",
    "        for i in range(0,self.n):\n",
    "            if board[col + i][row - i] == -1*player:\n",
    "                return 0\n",
    "            if board[col + i][row - i] == 0:\n",
    "                return i\n",
    "        return self.n\n",
    "    \n",
    "    def print_grid(self):\n",
    "        print(np.rot90(self.grid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def grid_to_key(grid):\n",
    "    \"\"\"\n",
    "    Converts ConnectN grid into string for dict indexing\n",
    "    \"\"\"\n",
    "\n",
    "    key = \"\"\n",
    "\n",
    "    for row in np.rot90(grid):\n",
    "        for column in row:\n",
    "            key += str(int(column))\n",
    "\n",
    "    return key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class ConnectDict(dict):\n",
    "    \"\"\"\n",
    "    Creates a Custom Dict that inherits from Python's native dict.\n",
    "    Takes in a number of states.\n",
    "    Adds keys to dict each time lookup is necessary to avoid full dict initialization.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, num_states, *arg, **kw):\n",
    "        self.num_states = num_states\n",
    "        super(ConnectDict, self).__init__(*arg, **kw)\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        if not dict.__contains__(self, key):\n",
    "            dict.__setitem__(self, key, np.zeros(self.num_states))\n",
    "        return dict.__getitem__(self, key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class TD_Learner(object):\n",
    "    \"\"\"\n",
    "    Base class for Temporal Difference Learners, like Sarsa and Q learning.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, task, value_table=None, epsilon=.1, discount_factor=.9, learning_rate=.5, player=1, trace_size=.1):\n",
    "        \n",
    "        self.num_states = task.grid_size\n",
    "        self.num_actions = task.grid_size\n",
    "        self.epsilon = epsilon\n",
    "        self.discount_factor = discount_factor\n",
    "        self.learning_rate = learning_rate\n",
    "        \n",
    "        if value_table == None:\n",
    "            self.value_table = ConnectDict(self.num_states)\n",
    "        else:\n",
    "            self.value_table = value_table\n",
    "            \n",
    "        self.e = ConnectDict(self.num_states)\n",
    "        self.player = player\n",
    "        self.trace_size = trace_size\n",
    "        self.last_board_state = None\n",
    "        self.last_action = None\n",
    "    \n",
    "    def softmax(self, next_board_state):\n",
    "        \"\"\"\n",
    "        Implementation of Softmax Policy, which weights towards better actions rather\n",
    "        than sampling uniformly across all possible actions (epsilon-greedy)\n",
    "        \"\"\"\n",
    "        \n",
    "        def weighted_pick(weights,n_picks):\n",
    "            t = np.cumsum(weights)\n",
    "            s = sum(weights)\n",
    "            return np.searchsorted(t,rand(n_picks)*s)\n",
    "        \n",
    "        tau = .5\n",
    "        key_val = grid_to_key(next_board_state.grid)\n",
    "        \n",
    "        vals = self.value_table[key_val]\n",
    "        num = ([math.e**(float(x)/tau) for x in vals])\n",
    "        \n",
    "        probs = [x/sum(num) for x in num]\n",
    "        best_action = weighted_pick(probs, 1)\n",
    "\n",
    "        return best_action[0]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_trial(agent, MIN_ITERATIONS, MIN_EPISODES, player):\n",
    "    \"\"\"\n",
    "    Runs the ConnectN Simulator for one player given an agent and number of iterations and episodes\n",
    "    \"\"\"\n",
    "    \n",
    "    rewards_by_iteration = []\n",
    "    rewards_by_episode = []\n",
    "    cumu_rewards_by_iteration = []\n",
    "    cumu_rewards_by_episode = []\n",
    "    \n",
    "    iteration = episode = 0\n",
    "    #agent.reset()\n",
    "    \n",
    "    new_value_table = None\n",
    "\n",
    "    while iteration < MIN_ITERATIONS or episode < MIN_EPISODES:\n",
    "        \n",
    "        task.reset()\n",
    "        board_state = task\n",
    "        reward = None\n",
    "        cumulative_reward = 0\n",
    "\n",
    "        while iteration < MIN_ITERATIONS or episode < MIN_EPISODES:\n",
    "                        \n",
    "            action = agent.interact(reward, board_state)\n",
    "\n",
    "            if task.move(action, player)[1] == 50:\n",
    "                print \"Won!\"\n",
    "                break\n",
    "\n",
    "            return_val, reward = task.move(action, player)\n",
    "\n",
    "            if iteration < MIN_ITERATIONS:\n",
    "                #print np.rot90(task.grid)\n",
    "\n",
    "                rewards_by_iteration.append(reward)\n",
    "                if cumu_rewards_by_iteration == []:\n",
    "                    cumu_rewards_by_iteration.append(reward)\n",
    "                else:\n",
    "                    cumu_rewards_by_iteration.append(cumu_rewards_by_iteration[-1] + reward)\n",
    "                \n",
    "            cumulative_reward += reward\n",
    "\n",
    "            iteration += 1\n",
    "\n",
    "        if episode < MIN_EPISODES:\n",
    "            rewards_by_episode.append(cumulative_reward)\n",
    "            if cumu_rewards_by_episode == []:\n",
    "                cumu_rewards_by_episode.append(cumulative_reward)\n",
    "            else:\n",
    "                cumu_rewards_by_episode.append(cumu_rewards_by_episode[-1] + cumulative_reward)\n",
    "        episode += 1\n",
    "        \n",
    "    return rewards_by_iteration, rewards_by_episode, cumu_rewards_by_iteration, cumu_rewards_by_episode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Q_Learner(TD_Learner):\n",
    "    \"\"\"\n",
    "    Implementation of Q Learning, inheriting from TD Learner base class. \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, task, value_table, known_states, epsilon=.1, discount_factor=.9, learning_rate=.5, player=1, trace_size=.1):   \n",
    "        TD_Learner.__init__(self, task, value_table, epsilon, discount_factor, learning_rate, player, trace_size) \n",
    "        self.known_states = known_states\n",
    "        \n",
    "\n",
    "    def interact(self, reward, next_board_state):\n",
    "        if reward is None:\n",
    "            # Approximation of known states. Since too many states, instead, given a board position, \n",
    "            # explore possible moves and give 15 reward to creating streaks of length 3 or 4 and \n",
    "            # 20 reward for preventing an opponent win.\n",
    "            if (self.known_states):\n",
    "                for col in task.next_possible_moves():\n",
    "                    row = np.sum([abs(x) for x in next_board_state.grid[col]])\n",
    "                    if next_board_state.streakVertical(next_board_state.grid, col, row - 2, self.player) >= 2:\n",
    "                        self.value_table[grid_to_key(next_board_state.grid)][col] = 15\n",
    "                    temp_board = deepcopy(next_board_state.grid)\n",
    "                    temp_board[col][row] = self.player\n",
    "                    for i in range(0, 4):\n",
    "                        if next_board_state.streakHorizontal(temp_board, col - i, row, self.player) >= 3:\n",
    "                            self.value_table[grid_to_key(next_board_state.grid)][col] = 15\n",
    "                        if next_board_state.streakDiagonalUp(temp_board, col - i, row - i, self.player) >= 3:\n",
    "                            self.value_table[grid_to_key(next_board_state.grid)][col] = 15\n",
    "                        if next_board_state.streakDiagonalDown(temp_board, col - i, row + i, self.player) >= 3:\n",
    "                            self.value_table[grid_to_key(next_board_state.grid)][col] = 15\n",
    "                            \n",
    "                    if next_board_state.streakVertical(next_board_state.grid, col, row - 3, -self.player) == 3:\n",
    "                        self.value_table[grid_to_key(next_board_state.grid)][col] = 20\n",
    "                    temp_board = deepcopy(next_board_state.grid)\n",
    "                    temp_board[col][row] = -1*self.player\n",
    "                    for i in range(0, 4):\n",
    "                        if next_board_state.streakHorizontal(temp_board, col - i, row, -1*self.player) == 4:\n",
    "                            self.value_table[grid_to_key(next_board_state.grid)][col] = 20\n",
    "                        if next_board_state.streakDiagonalUp(temp_board, col - i, row - i, -1*self.player) == 4:\n",
    "                            self.value_table[grid_to_key(next_board_state.grid)][col] = 20\n",
    "                        if next_board_state.streakDiagonalDown(temp_board, col - i, row + i, -1*self.player) == 4:\n",
    "                            self.value_table[grid_to_key(next_board_state.grid)][col] = 20\n",
    "\n",
    "            next_action = self.softmax(next_board_state)\n",
    "\n",
    "            self.last_board_state = next_board_state.grid\n",
    "            self.last_action = next_action\n",
    "            return self.last_action\n",
    "                \n",
    "        if reward == 50:\n",
    "            delta = delta = reward - self.value_table[grid_to_key(self.last_board_state)][self.last_action]\n",
    "            self.value_table[grid_to_key(self.last_board_state)][self.last_action] += self.learning_rate * delta\n",
    "            \n",
    "            return self.last_action\n",
    "        \n",
    "        \"\"\"\n",
    "        VDBE-Softmax policy. If draw < epsilon, perform Softmax. Else do best action.\n",
    "        \"\"\"\n",
    "        draw = np.random.uniform(0,1,1)\n",
    "\n",
    "        if draw < self.epsilon:\n",
    "            next_action = self.softmax(next_board_state)\n",
    "        else:\n",
    "            next_action = np.argmax(self.value_table[grid_to_key(next_board_state.grid)])\n",
    "\n",
    "        # Update value function.\n",
    "        delta = reward + self.discount_factor * np.amax(self.value_table[grid_to_key(next_board_state.grid)]) - self.value_table[grid_to_key(self.last_board_state)][self.last_action]\n",
    "        self.value_table[grid_to_key(self.last_board_state)][self.last_action] += self.learning_rate * delta\n",
    "        \n",
    "        # Update eligibility traces (Watson's Q(lambda))\n",
    "        self.e[grid_to_key(self.last_board_state)][self.last_action] += 1\n",
    "\n",
    "        # Eligibility traces\n",
    "        # Note that here we do not implement classic eligibility traces, which iterate over all state, action pairs\n",
    "        # Instead we consider all next possible board states and update those (for easier computation)\n",
    "        next_possible_moves = next_board_state.next_possible_moves()\n",
    "        next_possible_boards = []\n",
    "        \n",
    "        for i in next_possible_moves:\n",
    "            temp_board = deepcopy(next_board_state)\n",
    "            temp_board.move(next_action, self.player)\n",
    "            next_possible_boards.append(temp_board)\n",
    "            \n",
    "        for board in next_possible_boards:\n",
    "            valid_actions = board.next_possible_moves()\n",
    "            for action in valid_actions:\n",
    "                self.value_table[grid_to_key(board.grid)][action] += self.learning_rate * delta \\\n",
    "                                                                    * self.e[grid_to_key(board.grid)][action]\n",
    "                if self.last_action == action:\n",
    "                    self.e[grid_to_key(board.grid)][action] = self.discount_factor * self.trace_size \\\n",
    "                                                                    * self.e[grid_to_key(board.grid)][action]\n",
    "                else:\n",
    "                    self.e[grid_to_key(board.grid)][action] = 0\n",
    "                    \n",
    "        self.last_board_state = next_board_state.grid\n",
    "        self.last_action = next_action\n",
    "\n",
    "        if next_board_state.simulate_move(self.last_action, self.player) == 1:\n",
    "            self.last_action = self.softmax(next_board_state)\n",
    "            \n",
    "        return self.last_action\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "task = ConnectN(7, 4)\n",
    "agent = Q_Learner(task, None, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Won!\n",
      "Won!\n",
      "Won!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([0, 50, 0, 0, 50, 0, 0, 0, 50, 0],\n",
       " [50],\n",
       " [0, 50, 50, 50, 100, 100, 100, 100, 150, 150],\n",
       " [50])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_trial(agent, 10, 1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def play_game(board, p1, p2, q=False):\n",
    "    \"\"\"\n",
    "    Runs Connect 4 game given simulator object and two agents (players)\n",
    "    \"\"\"\n",
    "    reward = None\n",
    "    \n",
    "    if q == True:\n",
    "        while True:\n",
    "            print(\"p1\")\n",
    "            p1move = p1.interact(reward, board)\n",
    "            print(p1move)\n",
    "            if (p1move is None):\n",
    "                board.print_grid()\n",
    "                print(\"error player 1 a\")\n",
    "                return -1\n",
    "            p1result, reward = board.move(p1move, 1)\n",
    "            print p1result\n",
    "            if (p1result == 1):\n",
    "                print(\"player 1\")\n",
    "                board.print_grid()\n",
    "                return 1\n",
    "            elif (p1result == -1):\n",
    "                board.print_grid()\n",
    "                print(\"error player 1 b\")\n",
    "                return -1\n",
    "            print(\"p2\")\n",
    "            p2move = p2.calc_next_move()\n",
    "            print(p2move)\n",
    "            if (p2move is None):\n",
    "                board.print_grid()\n",
    "                print(\"error player 2\")\n",
    "                return -1\n",
    "            p2result = board.move(p2move, -1)\n",
    "            print p2result\n",
    "            if (p2result[0] == 1):\n",
    "                print(\"player 2\")\n",
    "                board.print_grid()\n",
    "                return 1\n",
    "            elif (p2result[0] == -1):\n",
    "                board.print_grid()\n",
    "                print(\"error player 2\")\n",
    "                return -1\n",
    "    \n",
    "    else:\n",
    "        while True:\n",
    "            print(\"p1\")\n",
    "            p1move = p1.calc_next_move()\n",
    "            print(p1move)\n",
    "            if (p1move is None):\n",
    "                board.print_grid()\n",
    "                print(\"error\")\n",
    "                return -1\n",
    "            p1result = board.move(p1move, 1)\n",
    "            print p1result\n",
    "            if (p1result[0] == 1):\n",
    "                print(\"player 1\")\n",
    "                board.print_grid()\n",
    "                return 1\n",
    "            elif (p1result[0] == -1):\n",
    "                board.print_grid()\n",
    "                print(\"error\")\n",
    "                return -1\n",
    "            print(\"p2\")\n",
    "            p2move = p2.calc_next_move()\n",
    "            print(p2move)\n",
    "            if (p2move is None):\n",
    "                board.print_grid()\n",
    "                print(\"error\")\n",
    "                return -1\n",
    "            p2result = board.move(p2move, -1)\n",
    "            print p2result\n",
    "            if (p2result[0] == 1):\n",
    "                print(\"player 2\")\n",
    "                board.print_grid()\n",
    "                return 1\n",
    "            elif (p2result[0] == -1):\n",
    "                board.print_grid()\n",
    "                print(\"error\")\n",
    "                return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Random_Learner(object):\n",
    "    \"\"\"\n",
    "    Implementation of Connect 4 agent that takes random moves at each action step\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, board):\n",
    "        self.board = board\n",
    "\n",
    "    def calc_next_move(self):\n",
    "        moves = self.board.next_possible_moves()\n",
    "        return moves[random.randint(0, len(moves) - 1)]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def play_game_no_output(board, p1, p2, q=False):\n",
    "    \"\"\"\n",
    "    Runs Connect 4 game given simulator object and two agents (players)\n",
    "    \"\"\"\n",
    "    reward = None\n",
    "    \n",
    "    if q == True:\n",
    "        while True:\n",
    "            p1move = p1.interact(reward, board)\n",
    "            if (p1move is None):\n",
    "                return -1\n",
    "            p1result, reward = board.move(p1move, 1)\n",
    "            if (p1result == 1):\n",
    "                p1.interact(reward, board)\n",
    "                return 1\n",
    "            elif (p1result == -1):\n",
    "                return -1\n",
    "            p2move = p2.calc_next_move()\n",
    "            if (p2move is None):\n",
    "                return -1\n",
    "            p2result = board.move(p2move, -1)\n",
    "            if (p2result[0] == 1):\n",
    "                return 2\n",
    "            elif (p2result[0] == -1):\n",
    "                return -1\n",
    "    \n",
    "    else:\n",
    "        while True:\n",
    "            print(\"p1\")\n",
    "            p1move = p1.calc_next_move()\n",
    "            print(p1move)\n",
    "            if (p1move is None):\n",
    "                board.print_grid()\n",
    "                print(\"error\")\n",
    "                return -1\n",
    "            p1result = board.move(p1move, 1)\n",
    "            print p1result\n",
    "            if (p1result[0] == 1):\n",
    "                print(\"player 1\")\n",
    "                board.print_grid()\n",
    "                return 1\n",
    "            elif (p1result[0] == -1):\n",
    "                board.print_grid()\n",
    "                print(\"error\")\n",
    "                return -1\n",
    "            print(\"p2\")\n",
    "            p2move = p2.calc_next_move()\n",
    "            print(p2move)\n",
    "            if (p2move is None):\n",
    "                board.print_grid()\n",
    "                print(\"error\")\n",
    "                return -1\n",
    "            p2result = board.move(p2move, -1)\n",
    "            print p2result\n",
    "            if (p2result[0] == 1):\n",
    "                print(\"player 2\")\n",
    "                board.print_grid()\n",
    "                return 1\n",
    "            elif (p2result[0] == -1):\n",
    "                board.print_grid()\n",
    "                print(\"error\")\n",
    "                return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def train_Q_learner(num_trials = 1000, k=2, n=4, grid_size = 7):\n",
    "    \"\"\"\n",
    "    Trains the Q Learner against Minimax Depth k\n",
    "    \n",
    "    Inputs:\n",
    "    Number of games to play\n",
    "    On grid_size x\n",
    "    N tokens to connect\n",
    "    \n",
    "    Outputs:\n",
    "    Q Learner value table after training\n",
    "    \"\"\"\n",
    "    depth = k\n",
    "    N = n\n",
    "    grid_size = grid_size\n",
    "    x = ConnectN(grid_size, N)\n",
    "    \n",
    "    p1 = Q_Learner(x, None, None, player=1)\n",
    "    p2 = Random_Learner(x)\n",
    "    play_game_no_output(x, p1, p2)\n",
    "    \n",
    "    for game in xrange(1, num_trials):\n",
    "                \n",
    "        x = ConnectN(grid_size, N)\n",
    "        p1 = Q_Learner(x, p1.value_table, None, player=1)\n",
    "        p2 = Random_Learner(x)\n",
    "        play_game_no_output(x, p1, p2)\n",
    "\n",
    "        if game == num_trials - 1:\n",
    "            return p1.value_table\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = ConnectN(7,4)\n",
    "x.move(0,1)\n",
    "x.move(1,-1)\n",
    "x.move(1,1)\n",
    "x.move(2,-1)\n",
    "x.move(2,-1)\n",
    "ql = Q_Learner(x, None, True)\n",
    "ql.interact(None, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'0000000000000000000000000000000000001-100001-1-10000': array([  0.,   0.,  15.,   0.,   0.,   0.,   0.])}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ql.value_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p1\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'Q_Learner' object has no attribute 'calc_next_move'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-3071e8df0898>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_Q_learner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_trials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-13-3f65e58b92fb>\u001b[0m in \u001b[0;36mtrain_Q_learner\u001b[0;34m(num_trials, k, n, grid_size)\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mp1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mQ_Learner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplayer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mp2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandom_Learner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m     \u001b[0mplay_game_no_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mgame\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_trials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-12-508129b55449>\u001b[0m in \u001b[0;36mplay_game_no_output\u001b[0;34m(board, p1, p2, q)\u001b[0m\n\u001b[1;32m     28\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m             \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"p1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m             \u001b[0mp1move\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcalc_next_move\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m             \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp1move\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mp1move\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Q_Learner' object has no attribute 'calc_next_move'"
     ]
    }
   ],
   "source": [
    "train_Q_learner(num_trials=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}