coco-experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from beam import BeamSearch\n",
    "from functools import partial\n",
    "from itertools import izip, repeat\n",
    "import sys\n",
    "import time\n",
    "sys.path.append('../coco-caption')\n",
    "sys.path.append('../show-attend-and-tell-tensorflow')\n",
    "from core.utils import load_pickle\n",
    "from pycocoevalcap.bleu.bleu import Bleu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded ../show-attend-and-tell-tensorflow/data/train/word_to_idx.pkl..\n",
      "Loaded ../show-attend-and-tell-tensorflow/data/train/train.references.pkl..\n"
     ]
    }
   ],
   "source": [
    "split = 'train'\n",
    "word_to_idx = load_pickle('../show-attend-and-tell-tensorflow/data/train/word_to_idx.pkl')\n",
    "words = word_to_idx.keys()\n",
    "words.remove('<START>')\n",
    "words.remove('<END>')\n",
    "words.remove('<NULL>')\n",
    "words.append('.')\n",
    "references = load_pickle('../show-attend-and-tell-tensorflow/data/{}/{}.references.pkl'.format(split, split))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Score function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scorer = Bleu(n=4)\n",
    "\n",
    "def score(hypotheses, reference_sequences):\n",
    "    \"\"\"\n",
    "    Score each hypothesis. Any sequence in reference_sequences is correct.\n",
    "    Returns the best score for each hypothesis, among the reference_sequences.\n",
    "    :param hypotheses: Dict of hypotheses to compute score.\n",
    "    :param reference_sequences: List of ground truth sequences.\n",
    "    :return scores: Dict with same keys as hypotheses, but with score as value\n",
    "    \"\"\"\n",
    "    #hypotheses = dict(izip(xrange(0, len(hypotheses)), hypotheses))\n",
    "    reference_sequences = dict(izip(hypotheses.iterkeys(), repeat(reference_sequences, len(hypotheses))))\n",
    "    _, scores = scorer.compute_score(reference_sequences, hypotheses)\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Beam Search\n",
    "#### with reference actions\n",
    "Reference words are as good, or better than, non-reference words in terms of BLEU score. Only a handful (~50) reference words to consider, but potentially 10k+ non-reference words. So by only considering the reference words as the set of actions available to expert, beam search is dramatically faster and produces the same results.\n",
    "\n",
    "~0.5s per search with reference word actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ground truth:\n",
      " \tcloseup of bins of food that include broccoli and bread .\n",
      " \ta meal is presented in brightly colored plastic trays .\n",
      " \tthere are containers filled with different kinds of foods .\n",
      " \ta bunch of trays that have different food .\n",
      " \tcolorful dishes holding meat vegetables fruit and bread .\n",
      "{'reflen': 1620, 'guess': [355, 175, 0, 0], 'testlen': 355, 'correct': [350, 7, 0, 0]}\n",
      "ratio: 0.219135802469\n",
      "Beam length 2 \n",
      " \t3.01973833959e-05: closeup of\n",
      " \t3.01973833959e-05: a bunch\n",
      " \t3.01973833959e-05: a meal\n",
      " \t3.01973833959e-05: there are\n",
      " \t3.01973833959e-05: a bunch\n",
      "{'reflen': 1575, 'guess': [525, 350, 175, 0], 'testlen': 525, 'correct': [516, 183, 5, 0]}\n",
      "ratio: 0.333333333333\n",
      "Beam length 3 \n",
      " \t0.00427967742473: closeup of bins\n",
      " \t0.00427967742473: a bunch of\n",
      " \t0.00427967742473: a meal is\n",
      " \t0.00427967742473: there are containers\n",
      " \t0.00427967742473: a bunch of\n",
      "{'reflen': 1575, 'guess': [700, 525, 350, 175], 'testlen': 700, 'correct': [688, 361, 180, 5]}\n",
      "ratio: 0.444444444444\n",
      "Beam length 4 \n",
      " \t0.286504796639: closeup of bins of\n",
      " \t0.286504796639: a bunch of trays\n",
      " \t0.286504796639: a meal is presented\n",
      " \t0.286504796639: there are containers filled\n",
      " \t0.286504796639: a bunch of trays\n",
      "{'reflen': 1575, 'guess': [875, 700, 525, 350], 'testlen': 875, 'correct': [858, 534, 355, 180]}\n",
      "ratio: 0.555555555555\n",
      "Beam length 5 \n",
      " \t0.449328963883: closeup of bins of food\n",
      " \t0.449328963883: a bunch of trays that\n",
      " \t0.449328963883: a meal is presented in\n",
      " \t0.449328963883: there are containers filled with\n",
      " \t0.449328963883: a bunch of trays that\n",
      "{'reflen': 1575, 'guess': [1050, 875, 700, 525], 'testlen': 1050, 'correct': [1028, 708, 530, 355]}\n",
      "ratio: 0.666666666666\n",
      "Beam length 6 \n",
      " \t0.606530659467: closeup of bins of food that\n",
      " \t0.606530659467: a bunch of trays that have\n",
      " \t0.606530659467: a meal is presented in brightly\n",
      " \t0.606530659467: there are containers filled with different\n",
      " \t0.606530659467: a bunch of trays that have\n",
      "{'reflen': 1575, 'guess': [1225, 1050, 875, 700], 'testlen': 1225, 'correct': [1198, 882, 705, 530]}\n",
      "ratio: 0.777777777777\n",
      "Beam length 7 \n",
      " \t0.751477292825: closeup of bins of food that include\n",
      " \t0.751477292825: a bunch of trays that have different\n",
      " \t0.751477292825: a meal is presented in brightly colored\n",
      " \t0.751477292825: there are containers filled with different kinds\n",
      " \t0.751477292825: a bunch of trays that have different\n",
      "{'reflen': 1575, 'guess': [1400, 1225, 1050, 875], 'testlen': 1400, 'correct': [1368, 1057, 880, 705]}\n",
      "ratio: 0.888888888888\n",
      "Beam length 8 \n",
      " \t0.882496902334: closeup of bins of food that include broccoli\n",
      " \t0.882496902334: a bunch of trays that have different food\n",
      " \t0.882496902334: a meal is presented in brightly colored plastic\n",
      " \t0.882496902334: there are containers filled with different kinds of\n",
      " \t0.882496902334: a bunch of trays that have different food\n",
      "{'reflen': 1575, 'guess': [1575, 1400, 1225, 1050], 'testlen': 1575, 'correct': [1539, 1235, 1055, 880]}\n",
      "ratio: 0.999999999999\n",
      "Beam length 9 \n",
      " \t0.999999999752: closeup of bins of food that include broccoli and\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      " \t0.999999999752: a meal is presented in brightly colored plastic trays\n",
      " \t0.999999999752: there are containers filled with different kinds of foods\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      "{'reflen': 1068, 'guess': [1068, 961, 854, 747], 'testlen': 1068, 'correct': [1043, 860, 752, 645]}\n",
      "ratio: 0.999999999999\n",
      "Beam length 10 \n",
      " \t0.99999999978: closeup of bins of food that include broccoli and bread\n",
      " \t0.99999999978: a meal is presented in brightly colored plastic trays .\n",
      " \t0.99999999978: there are containers filled with different kinds of foods .\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      "{'reflen': 423, 'guess': [423, 384, 345, 306], 'testlen': 423, 'correct': [414, 350, 311, 272]}\n",
      "ratio: 0.999999999998\n",
      "Beam length 11 \n",
      " \t0.999999999802: closeup of bins of food that include broccoli and bread .\n",
      " \t0.99999999978: a meal is presented in brightly colored plastic trays .\n",
      " \t0.99999999978: there are containers filled with different kinds of foods .\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      "Ground truth:\n",
      " \ta giraffe standing next to a forest filled with trees .\n",
      " \ta giraffe mother with its baby in the forest .\n",
      " \ttwo giraffes standing in a tree filled area .\n",
      " \ta giraffe standing up nearby a tree .\n",
      " \ta giraffe eating food from the top of the tree .\n",
      "{'reflen': 1080, 'guess': [265, 130, 0, 0], 'testlen': 265, 'correct': [264, 13, 0, 0]}\n",
      "ratio: 0.24537037037\n",
      "Beam length 2 \n",
      " \t4.97870683243e-05: a forest\n",
      " \t4.97870683243e-05: a giraffe\n",
      " \t4.97870683243e-05: a tree\n",
      " \t4.97870683243e-05: a forest\n",
      " \t4.97870683243e-05: a giraffe\n",
      "{'reflen': 1040, 'guess': [390, 260, 130, 0], 'testlen': 390, 'correct': [385, 142, 10, 0]}\n",
      "ratio: 0.375\n",
      "Beam length 3 \n",
      " \t0.00597277098931: a forest filled\n",
      " \t0.00597277098931: a giraffe standing\n",
      " \t0.00597277098931: a giraffe eating\n",
      " \t0.00597277098931: a giraffe mother\n",
      " \t0.00597277098931: a tree .\n",
      "{'reflen': 840, 'guess': [419, 314, 209, 104], 'testlen': 419, 'correct': [411, 217, 110, 5]}\n",
      "ratio: 0.498809523809\n",
      "Beam length 4 \n",
      " \t0.367879440888: a forest filled with\n",
      " \t0.367879440888: a giraffe standing next\n",
      " \t0.367879440888: a giraffe standing up\n",
      " \t0.367879440888: a giraffe eating food\n",
      " \t0.367879440888: a giraffe mother with\n",
      "{'reflen': 1040, 'guess': [650, 520, 390, 260], 'testlen': 650, 'correct': [635, 397, 265, 135]}\n",
      "ratio: 0.624999999999\n",
      "Beam length 5 \n",
      " \t0.548811635808: a forest filled with trees\n",
      " \t0.548811635808: a giraffe standing next to\n",
      " \t0.548811635808: a giraffe standing up nearby\n",
      " \t0.548811635808: a giraffe eating food from\n",
      " \t0.548811635808: a giraffe mother with its\n",
      "{'reflen': 1040, 'guess': [780, 650, 520, 390], 'testlen': 780, 'correct': [760, 525, 395, 265]}\n",
      "ratio: 0.749999999999\n",
      "Beam length 6 \n",
      " \t0.716531310284: a forest filled with trees .\n",
      " \t0.716531310284: a giraffe standing next to a\n",
      " \t0.716531310284: a giraffe standing up nearby a\n",
      " \t0.716531310284: a giraffe eating food from the\n",
      " \t0.716531310284: a giraffe mother with its baby\n",
      "{'reflen': 840, 'guess': [734, 629, 524, 419], 'testlen': 734, 'correct': [715, 533, 424, 319]}\n",
      "ratio: 0.873809523808\n",
      "Beam length 7 \n",
      " \t0.866877899462: a giraffe standing next to a forest\n",
      " \t0.866877899462: a giraffe standing up nearby a tree\n",
      " \t0.866877899462: a giraffe eating food from the top\n",
      " \t0.866877899462: a giraffe mother with its baby in\n",
      " \t0.762949413683: a giraffe standing next to a tree\n",
      "{'reflen': 1040, 'guess': [1040, 910, 780, 650], 'testlen': 1040, 'correct': [1011, 789, 631, 498]}\n",
      "ratio: 0.999999999999\n",
      "Beam length 8 \n",
      " \t0.999999999716: a giraffe standing next to a forest filled\n",
      " \t0.999999999716: a giraffe standing up nearby a tree .\n",
      " \t0.999999999716: a giraffe eating food from the top of\n",
      " \t0.999999999716: a giraffe mother with its baby in the\n",
      " \t0.945741608735: a giraffe standing up nearby a tree filled\n",
      "{'reflen': 944, 'guess': [944, 839, 734, 629], 'testlen': 944, 'correct': [918, 743, 634, 503]}\n",
      "ratio: 0.999999999999\n",
      "Beam length 9 \n",
      " \t0.999999999752: a giraffe standing next to a forest filled with\n",
      " \t0.999999999752: a giraffe eating food from the top of the\n",
      " \t0.999999999752: a giraffe mother with its baby in the forest\n",
      " \t0.999999999716: a giraffe standing up nearby a tree .\n",
      " \t0.955442791968: a giraffe standing up nearby a tree filled area\n",
      "{'reflen': 1048, 'guess': [1048, 943, 838, 733], 'testlen': 1048, 'correct': [1018, 846, 738, 607]}\n",
      "ratio: 0.999999999999\n",
      "Beam length 10 \n",
      " \t0.99999999978: a giraffe standing next to a forest filled with trees\n",
      " \t0.99999999978: a giraffe eating food from the top of the tree\n",
      " \t0.99999999978: a giraffe mother with its baby in the forest .\n",
      " \t0.999999999716: a giraffe standing up nearby a tree .\n",
      " \t0.962195457984: a giraffe standing up nearby a tree filled area .\n",
      "{'reflen': 600, 'guess': [600, 545, 490, 435], 'testlen': 600, 'correct': [583, 496, 440, 384]}\n",
      "ratio: 0.999999999998\n",
      "Beam length 11 \n",
      " \t0.999999999802: a giraffe standing next to a forest filled with trees .\n",
      " \t0.999999999802: a giraffe eating food from the top of the tree .\n",
      " \t0.99999999978: a giraffe mother with its baby in the forest .\n",
      " \t0.999999999716: a giraffe standing up nearby a tree .\n",
      " \t0.962195457984: a giraffe standing up nearby a tree filled area .\n",
      " \n",
      " Elapsed time per search: 0.505202531815s \n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "n_beams = 5\n",
    "n_searches = 2\n",
    "for counter, (_, reference) in enumerate(references.iteritems()):\n",
    "    if counter >= n_searches:\n",
    "        break\n",
    "\n",
    "    print 'Ground truth:\\n \\t', '\\n \\t'.join(reference)\n",
    "    score_wrapper = partial(score, reference_sequences=reference)\n",
    "    reference_words = set()\n",
    "    for sentence in reference:\n",
    "        reference_words.update(sentence.split(' '))\n",
    "    beam_search = BeamSearch(n_beams, score_wrapper, reference_words)\n",
    "    initial_beams = [[sentence.split(' ')[0]] for sentence in reference]\n",
    "    scores, sequence = beam_search.search(max_length=15, beams=initial_beams)\n",
    "\n",
    "end_time = time.time()\n",
    "print ' \\n Elapsed time per search: {}s \\n \\n'.format(float(end_time - start_time) / n_searches)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With all actions\n",
    "~360s per search with reference word actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ground truth:\n",
      " \tcloseup of bins of food that include broccoli and bread .\n",
      " \ta meal is presented in brightly colored plastic trays .\n",
      " \tthere are containers filled with different kinds of foods .\n",
      " \ta bunch of trays that have different food .\n",
      " \tcolorful dishes holding meat vegetables fruit and bread .\n",
      "{'reflen': 1039905, 'guess': [231085, 115540, 0, 0], 'testlen': 231085, 'correct': [115715, 7, 0, 0]}\n",
      "ratio: 0.222217414091\n",
      "Beam length 2 \n",
      " \t3.01973833959e-05: closeup of\n",
      " \t3.01973833959e-05: a bunch\n",
      " \t3.01973833959e-05: a meal\n",
      " \t3.01973833959e-05: there are\n",
      " \t3.01973833959e-05: a bunch\n",
      "{'reflen': 1039860, 'guess': [346620, 231080, 115540, 0], 'testlen': 346620, 'correct': [231246, 115548, 5, 0]}\n",
      "ratio: 0.333333333333\n",
      "Beam length 3 \n",
      " \t0.00427967742473: closeup of bins\n",
      " \t0.00427967742473: a bunch of\n",
      " \t0.00427967742473: a meal is\n",
      " \t0.00427967742473: there are containers\n",
      " \t0.00427967742473: a bunch of\n",
      "{'reflen': 1039860, 'guess': [462160, 346620, 231080, 115540], 'testlen': 462160, 'correct': [346783, 231091, 115545, 5]}\n",
      "ratio: 0.444444444444\n",
      "Beam length 4 \n",
      " \t0.286504796639: closeup of bins of\n",
      " \t0.286504796639: a bunch of trays\n",
      " \t0.286504796639: a meal is presented\n",
      " \t0.286504796639: there are containers filled\n",
      " \t0.286504796639: a bunch of trays\n",
      "{'reflen': 1039860, 'guess': [577700, 462160, 346620, 231080], 'testlen': 577700, 'correct': [462318, 346629, 231085, 115545]}\n",
      "ratio: 0.555555555556\n",
      "Beam length 5 \n",
      " \t0.449328963883: closeup of bins of food\n",
      " \t0.449328963883: a bunch of trays that\n",
      " \t0.449328963883: a meal is presented in\n",
      " \t0.449328963883: there are containers filled with\n",
      " \t0.449328963883: a bunch of trays that\n",
      "{'reflen': 1039860, 'guess': [693240, 577700, 462160, 346620], 'testlen': 693240, 'correct': [577853, 462168, 346625, 231085]}\n",
      "ratio: 0.666666666667\n",
      "Beam length 6 \n",
      " \t0.606530659467: closeup of bins of food that\n",
      " \t0.606530659467: a bunch of trays that have\n",
      " \t0.606530659467: a meal is presented in brightly\n",
      " \t0.606530659467: there are containers filled with different\n",
      " \t0.606530659467: a bunch of trays that have\n",
      "{'reflen': 1039860, 'guess': [808780, 693240, 577700, 462160], 'testlen': 808780, 'correct': [693388, 577707, 462165, 346625]}\n",
      "ratio: 0.777777777778\n",
      "Beam length 7 \n",
      " \t0.751477292825: closeup of bins of food that include\n",
      " \t0.751477292825: a bunch of trays that have different\n",
      " \t0.751477292825: a meal is presented in brightly colored\n",
      " \t0.751477292825: there are containers filled with different kinds\n",
      " \t0.751477292825: a bunch of trays that have different\n",
      "{'reflen': 1039860, 'guess': [924320, 808780, 693240, 577700], 'testlen': 924320, 'correct': [808923, 693247, 577705, 462165]}\n",
      "ratio: 0.888888888889\n",
      "Beam length 8 \n",
      " \t0.882496902334: closeup of bins of food that include broccoli\n",
      " \t0.882496902334: a bunch of trays that have different food\n",
      " \t0.882496902334: a meal is presented in brightly colored plastic\n",
      " \t0.882496902334: there are containers filled with different kinds of\n",
      " \t0.882496902334: a bunch of trays that have different food\n",
      "{'reflen': 1039860, 'guess': [1039860, 924320, 808780, 693240], 'testlen': 1039860, 'correct': [924459, 808790, 693245, 577705]}\n",
      "ratio: 1.0\n",
      "Beam length 9 \n",
      " \t0.999999999752: closeup of bins of food that include broccoli and\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      " \t0.999999999752: a meal is presented in brightly colored plastic trays\n",
      " \t0.999999999752: there are containers filled with different kinds of foods\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      "{'reflen': 693258, 'guess': [693258, 623932, 554606, 485280], 'testlen': 693258, 'correct': [624014, 554612, 485285, 415959]}\n",
      "ratio: 1.0\n",
      "Beam length 10 \n",
      " \t0.99999999978: closeup of bins of food that include broccoli and bread\n",
      " \t0.99999999978: a meal is presented in brightly colored plastic trays .\n",
      " \t0.99999999978: there are containers filled with different kinds of foods .\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      "{'reflen': 254226, 'guess': [254226, 231114, 208002, 184890], 'testlen': 254226, 'correct': [231144, 208007, 184895, 161783]}\n",
      "ratio: 1.0\n",
      "Beam length 11 \n",
      " \t0.999999999802: closeup of bins of food that include broccoli and bread .\n",
      " \t0.99999999978: a meal is presented in brightly colored plastic trays .\n",
      " \t0.99999999978: there are containers filled with different kinds of foods .\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      " \t0.999999999752: a bunch of trays that have different food .\n",
      "Ground truth:\n",
      " \ta giraffe standing next to a forest filled with trees .\n",
      " \ta giraffe mother with its baby in the forest .\n",
      " \ttwo giraffes standing in a tree filled area .\n",
      " \ta giraffe standing up nearby a tree .\n",
      " \ta giraffe eating food from the top of the tree .\n",
      "{'reflen': 924360, 'guess': [231085, 115540, 0, 0], 'testlen': 231085, 'correct': [115674, 13, 0, 0]}\n",
      "ratio: 0.249994590852\n",
      "Beam length 2 \n",
      " \t4.97870683243e-05: a giraffe\n",
      " \t4.97870683243e-05: a tree\n",
      " \t4.97870683243e-05: a forest\n",
      " \t4.97870683243e-05: a giraffe\n",
      " \t4.97870683243e-05: a tree\n",
      "{'reflen': 924320, 'guess': [346620, 231080, 115540, 0], 'testlen': 346620, 'correct': [231205, 115552, 11, 0]}\n",
      "ratio: 0.375\n",
      "Beam length 3 \n",
      " \t0.00597277098931: a giraffe standing\n",
      " \t0.00597277098931: a giraffe mother\n",
      " \t0.00597277098931: a giraffe eating\n",
      " \t0.00597277098931: a tree filled\n",
      " \t0.00597277098931: a tree .\n",
      "{'reflen': 739464, 'guess': [369731, 277298, 184865, 92432], 'testlen': 369731, 'correct': [277395, 184873, 92438, 5]}\n",
      "ratio: 0.499998647669\n",
      "Beam length 4 \n",
      " \t0.367879440888: a giraffe standing next\n",
      " \t0.367879440888: a giraffe standing up\n",
      " \t0.367879440888: a giraffe mother with\n",
      " \t0.367879440888: a giraffe eating food\n",
      " \t0.367879440888: a tree filled area\n",
      "{'reflen': 924320, 'guess': [577700, 462160, 346620, 231080], 'testlen': 577700, 'correct': [462275, 346626, 231085, 115545]}\n",
      "ratio: 0.625\n",
      "Beam length 5 \n",
      " \t0.548811635808: a giraffe standing next to\n",
      " \t0.548811635808: a giraffe standing up nearby\n",
      " \t0.548811635808: a giraffe mother with its\n",
      " \t0.548811635808: a giraffe eating food from\n",
      " \t0.548811635808: a tree filled area .\n",
      "{'reflen': 739464, 'guess': [554597, 462164, 369731, 277298], 'testlen': 554597, 'correct': [462253, 369736, 277303, 184870]}\n",
      "ratio: 0.749998647669\n",
      "Beam length 6 \n",
      " \t0.716531310284: a giraffe standing next to a\n",
      " \t0.716531310284: a giraffe standing up nearby a\n",
      " \t0.716531310284: a giraffe mother with its baby\n",
      " \t0.716531310284: a giraffe eating food from the\n",
      " \t0.569836377314: a giraffe standing next to baby\n",
      "{'reflen': 924320, 'guess': [808780, 693240, 577700, 462160], 'testlen': 808780, 'correct': [693346, 554601, 439056, 323516]}\n",
      "ratio: 0.875\n",
      "Beam length 7 \n",
      " \t0.866877899462: a giraffe standing next to a forest\n",
      " \t0.866877899462: a giraffe standing up nearby a tree\n",
      " \t0.866877899462: a giraffe mother with its baby in\n",
      " \t0.866877899462: a giraffe eating food from the top\n",
      " \t0.762949413683: a giraffe standing next to a tree\n",
      "{'reflen': 924320, 'guess': [924320, 808780, 693240, 577700], 'testlen': 924320, 'correct': [808881, 693249, 554599, 439056]}\n",
      "ratio: 1.0\n",
      "Beam length 8 \n",
      " \t0.999999999716: a giraffe standing next to a forest filled\n",
      " \t0.999999999716: a giraffe standing up nearby a tree .\n",
      " \t0.999999999716: a giraffe mother with its baby in the\n",
      " \t0.999999999716: a giraffe eating food from the top of\n",
      " \t0.945741608735: a giraffe standing up nearby a tree filled\n",
      "{'reflen': 831896, 'guess': [831896, 739463, 647030, 554597], 'testlen': 831896, 'correct': [739542, 647039, 554602, 439061]}\n",
      "ratio: 1.0\n",
      "Beam length 9 \n",
      " \t0.999999999752: a giraffe standing next to a forest filled with\n",
      " \t0.999999999752: a giraffe mother with its baby in the forest\n",
      " \t0.999999999752: a giraffe eating food from the top of the\n",
      " \t0.999999999716: a giraffe standing up nearby a tree .\n",
      " \t0.955442791968: a giraffe standing up nearby a tree filled area\n",
      "{'reflen': 924328, 'guess': [924328, 831895, 739462, 647029], 'testlen': 924328, 'correct': [831970, 739470, 647034, 531493]}\n",
      "ratio: 1.0\n",
      "Beam length 10 \n",
      " \t0.99999999978: a giraffe standing next to a forest filled with trees\n",
      " \t0.99999999978: a giraffe mother with its baby in the forest .\n",
      " \t0.99999999978: a giraffe eating food from the top of the tree\n",
      " \t0.999999999716: a giraffe standing up nearby a tree .\n",
      " \t0.962195457984: a giraffe standing up nearby a tree filled area .\n",
      "{'reflen': 508404, 'guess': [508404, 462185, 415966, 369747], 'testlen': 508404, 'correct': [462223, 415972, 369752, 323532]}\n",
      "ratio: 1.0\n",
      "Beam length 11 \n",
      " \t0.999999999802: a giraffe standing next to a forest filled with trees .\n",
      " \t0.999999999802: a giraffe eating food from the top of the tree .\n",
      " \t0.99999999978: a giraffe mother with its baby in the forest .\n",
      " \t0.999999999716: a giraffe standing up nearby a tree .\n",
      " \t0.962195457984: a giraffe standing up nearby a tree filled area .\n",
      " \n",
      " Elapsed time per search: 362.784044504s \n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "n_beams = 5\n",
    "n_searches = 2\n",
    "for counter, (_, reference) in enumerate(references.iteritems()):\n",
    "    if counter >= n_searches:\n",
    "        break\n",
    "\n",
    "    print 'Ground truth:\\n \\t', '\\n \\t'.join(reference)\n",
    "    score_wrapper = partial(score, reference_sequences=reference)\n",
    "    beam_search = BeamSearch(n_beams, score_wrapper, words)\n",
    "    initial_beams = [[sentence.split(' ')[0]] for sentence in reference]\n",
    "    scores, sequence = beam_search.search(max_length=15, beams=initial_beams)\n",
    "\n",
    "end_time = time.time()\n",
    "print ' \\n Elapsed time per search: {}s \\n \\n'.format(float(end_time - start_time) / n_searches)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}