eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import cv2  # OpenCVライブラリ\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "\n",
    "import pickle\n",
    "\n",
    "import torch.utils.data as data\n",
    "from itertools import product as product\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dataset\n",
    "from utils.dataset import VOCDataset, COCODatasetTransform, make_datapath_list, Anno_xml2list, od_collate_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set your VOCdevkit path!\n",
    "vocpath = \"../VOCdevkit/VOC2007\"\n",
    "DEVKIT_PATH = \"../VOCdevkit/\"\n",
    "SET = \"test\"\n",
    "train_img_list, train_anno_list, val_img_list, val_anno_list = make_datapath_list(vocpath)\n",
    "\n",
    "model=\"efficientdet\"\n",
    "backbone = \"efficientnet-b0\"\n",
    "scale = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../VOCdevkit/VOC2007/JPEGImages/000001.jpg'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_img_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'000001'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_index = []\n",
    "for l in val_img_list:\n",
    "    image_index.append(l[-10:-4])\n",
    "\n",
    "image_index[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names = ['aeroplane', 'bicycle', 'bird', 'boat',\n",
    "               'bottle', 'bus', 'car', 'cat', 'chair',\n",
    "               'cow', 'diningtable', 'dog', 'horse',\n",
    "               'motorbike', 'person', 'pottedplant',\n",
    "               'sheep', 'sofa', 'train', 'tvmonitor']\n",
    "color_mean = (104, 117, 123)  # (BGR)の色の平均値\n",
    "if scale == 1:\n",
    "    input_size = 300  # 画像のinputサイズを300×300にする\n",
    "elif scale == 2:\n",
    "    input_size = 512\n",
    "\n",
    "## DatasetTransformを適応\n",
    "#transform = DatasetTransform(input_size, color_mean)\n",
    "#transform_anno = Anno_xml2list(class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#val_dataset = VOCDataset(val_img_list, val_anno_list, phase=\"val\", transform=COCODatasetTransform(\n",
    "#    input_size, color_mean), transform_anno=Anno_xml2list(class_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#val_dataloader = data.DataLoader(\n",
    "#    val_dataset, batch_size=1, shuffle=False, collate_fn=od_collate_fn, num_workers=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# set up model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b0\n",
      "320\n",
      "use BiFPN\n",
      "loaded the trained weights\n",
      "using: cuda:0\n"
     ]
    }
   ],
   "source": [
    "if model==\"SSD\":\n",
    "    from utils.ssd_model import SSD\n",
    "elif model==\"retina\":\n",
    "    from utils.retinanet import RetinaFPN as SSD\n",
    "    from utils.retinanet import Bottleneck\n",
    "elif model==\"efficientdet\":\n",
    "    from utils.efficientdet import EfficientDet\n",
    "    \n",
    "voc_classes = ['aeroplane', 'bicycle', 'bird', 'boat',\n",
    "               'bottle', 'bus', 'car', 'cat', 'chair',\n",
    "               'cow', 'diningtable', 'dog', 'horse',\n",
    "               'motorbike', 'person', 'pottedplant',\n",
    "               'sheep', 'sofa', 'train', 'tvmonitor']\n",
    "\n",
    "if scale==1:\n",
    "    ssd_cfg = {\n",
    "        'num_classes': 21,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 300*scale,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6, 6, 6, 4, 4],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [37, 18, 9, 5, 3, 1],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16, 32, 64, 100, 300],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60, 111, 162, 213, 264],  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 111, 162, 213, 264, 315],  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "elif scale==2:\n",
    "    ssd_cfg = {\n",
    "        'num_classes': 21,  # 背景クラスを含めた合計クラス数\n",
    "        'input_size': 512,  # 画像の入力サイズ\n",
    "        'bbox_aspect_num': [4, 6, 6, 6, 4, 4],  # 出力するDBoxのアスペクト比の種類\n",
    "        'feature_maps': [64, 32, 16, 8, 4, 2],  # 各sourceの画像サイズ\n",
    "        'steps': [8, 16, 32, 64, 100, 300],  # DBOXの大きさを決める\n",
    "        'min_sizes': [30, 60, 111, 162, 213, 264]*scale,  # DBOXの大きさを決める\n",
    "        'max_sizes': [60, 111, 162, 213, 264, 315]*scale,  # DBOXの大きさを決める\n",
    "        'aspect_ratios': [[2], [2, 3], [2, 3], [2, 3], [2], [2]],\n",
    "    }\n",
    "\n",
    "# SSDネットワークモデル\n",
    "if model==\"SSD\":\n",
    "    net = SSD(phase=\"inference\", cfg=ssd_cfg).eval()\n",
    "    net_weights = torch.load('./weights/ssd300_200.pth',\n",
    "                         map_location={'cuda:0': 'cpu'})\n",
    "elif model==\"retina\":\n",
    "    net = SSD(Bottleneck, [2,2,2,2], phase=\"inference\", cfg=ssd_cfg, model=backbone).to(\"cuda\")\n",
    "    net_weights = torch.load('./weights/retinanet300_200.pth',\n",
    "                         map_location={'cuda:0': 'cpu'})\n",
    "else:\n",
    "    net = EfficientDet(phase=\"inference\", cfg=ssd_cfg, verbose=False, backbone=backbone, useBiFPN=True)\n",
    "    net_weights = torch.load('./weights/VOC_efficientnet-b0_300_BiFPN_200.pth',\n",
    "                         map_location={'cuda:0': 'cpu'})\n",
    "\n",
    "net.load_state_dict(net_weights)\n",
    "\n",
    "print('loaded the trained weights')\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"using:\", device)\n",
    "\n",
    "net = net.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_imgs = []\n",
    "classes = {}\n",
    "bbox_threshold = 0.05\n",
    "\n",
    "# define detections\n",
    "all_boxes = [[[] for _ in range(len(val_img_list))]\n",
    "               for _ in range(21)]\n",
    "empty_array = np.transpose(np.array([[],[],[],[],[]]), (1,0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "from utils.ssd_predict_show import SSDPredictShow\n",
    "ssd = SSDPredictShow(eval_categories=voc_classes, net=net, device=device, image_size=input_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# infer images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#all_boxes = ssd.ssd_inference(val_dataloader, all_boxes, data_confidence_level=bbox_threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../VOCdevkit/VOC2007/JPEGImages/000001.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000002.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000003.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000004.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000006.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000008.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000010.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000011.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000013.jpg',\n",
       " '../VOCdevkit/VOC2007/JPEGImages/000014.jpg']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_img_list[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iter: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2457: UserWarning: nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.functional.upsample is deprecated. Use nn.functional.interpolate instead.\")\n",
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py:2539: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iter: 1000\n",
      "iter: 3000\n",
      "iter: 4000\n"
     ]
    }
   ],
   "source": [
    "for i, imp in enumerate(val_img_list):\n",
    "    detections, pre_dict_label_index = ssd.ssd_predict2(imp, data_confidence_level=0.05)\n",
    "    \n",
    "    for cls in range(21):\n",
    "        box = []\n",
    "        for j,pred in enumerate(detections):\n",
    "            if cls == pre_dict_label_index[j]:\n",
    "                box.append(pred)\n",
    "        if not box == []:\n",
    "            all_boxes[cls][i] = box\n",
    "        else:\n",
    "            all_boxes[cls][i] = empty_array\n",
    "    if i%1000==0:\n",
    "        print(\"iter:\", i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([], shape=(0, 5), dtype=float64),\n",
       " array([], shape=(0, 5), dtype=float64),\n",
       " array([], shape=(0, 5), dtype=float64),\n",
       " array([], shape=(0, 5), dtype=float64),\n",
       " array([], shape=(0, 5), dtype=float64),\n",
       " array([], shape=(0, 5), dtype=float64),\n",
       " array([], shape=(0, 5), dtype=float64),\n",
       " [array([  0.7535214, 137.66829  ,  84.52406  , 326.27124  , 302.4893   ],\n",
       "        dtype=float32)],\n",
       " array([], shape=(0, 5), dtype=float64),\n",
       " array([], shape=(0, 5), dtype=float64)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_boxes[7][0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# eval accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# eval function\n",
    "def voc_eval(detpath,\n",
    "             annopath,\n",
    "             imagesetfile,\n",
    "             classname,\n",
    "             cachedir,\n",
    "             ovthresh=0.5,\n",
    "             use_07_metric=False):\n",
    "  \"\"\"\n",
    "  rec, prec, ap = voc_eval(detpath,\n",
    "                              annopath,\n",
    "                              imagesetfile,\n",
    "                              classname,\n",
    "                              [ovthresh],\n",
    "                              [use_07_metric])\n",
    "  Top level function that does the PASCAL VOC evaluation.\n",
    "  detpath: Path to detections\n",
    "      detpath.format(classname) should produce the detection results file.\n",
    "  annopath: Path to annotations\n",
    "      annopath.format(imagename) should be the xml annotations file.\n",
    "  imagesetfile: Text file containing the list of images, one image per line.\n",
    "  classname: Category name (duh)\n",
    "  cachedir: Directory for caching the annotations\n",
    "  [ovthresh]: Overlap threshold (default = 0.5)\n",
    "  [use_07_metric]: Whether to use VOC07's 11 point AP computation\n",
    "      (default False)\n",
    "  \"\"\"\n",
    "  # assumes detections are in detpath.format(classname)\n",
    "  # assumes annotations are in annopath.format(imagename)\n",
    "  # assumes imagesetfile is a text file with each line an image name\n",
    "  # cachedir caches the annotations in a pickle file\n",
    "\n",
    "  # first load gt\n",
    "  if not os.path.isdir(cachedir):\n",
    "    os.mkdir(cachedir)\n",
    "  cachefile = os.path.join(cachedir, '%s_annots.pkl' % imagesetfile)\n",
    "  # read list of images\n",
    "  with open(imagesetfile, 'r') as f:\n",
    "    lines = f.readlines()\n",
    "  imagenames = [x.strip() for x in lines]\n",
    "\n",
    "  if not os.path.isfile(cachefile):\n",
    "    # load annotations\n",
    "    recs = {}\n",
    "    for i, imagename in enumerate(imagenames):\n",
    "      recs[imagename] = parse_rec(annopath.format(imagename))\n",
    "      #if i % 100 == 0:\n",
    "        #print('Reading annotation for {:d}/{:d}'.format(\n",
    "        #  i + 1, len(imagenames)))\n",
    "    # save\n",
    "    #print('Saving cached annotations to {:s}'.format(cachefile))\n",
    "    #with open(cachefile, 'wb') as f:\n",
    "    #  pickle.dump(recs, f)\n",
    "  else:\n",
    "    # load\n",
    "    with open(cachefile, 'rb') as f:\n",
    "      try:\n",
    "        recs = pickle.load(f)\n",
    "      except:\n",
    "        recs = pickle.load(f, encoding='bytes')\n",
    "\n",
    "  # extract gt objects for this class\n",
    "  class_recs = {}\n",
    "  npos = 0\n",
    "  for imagename in imagenames:\n",
    "    R = [obj for obj in recs[imagename] if obj['name'] == classname]\n",
    "    bbox = np.array([x['bbox'] for x in R])\n",
    "    difficult = np.array([x['difficult'] for x in R]).astype(np.bool)\n",
    "    det = [False] * len(R)\n",
    "    npos = npos + sum(~difficult)\n",
    "    class_recs[imagename] = {'bbox': bbox,\n",
    "                             'difficult': difficult,\n",
    "                             'det': det}\n",
    "\n",
    "  # read dets\n",
    "  detfile = detpath.format(classname)\n",
    "  with open(detfile, 'r') as f:\n",
    "    lines = f.readlines()\n",
    "\n",
    "  splitlines = [x.strip().split(' ') for x in lines]\n",
    "  image_ids = [x[0] for x in splitlines]\n",
    "  confidence = np.array([float(x[1]) for x in splitlines])\n",
    "  BB = np.array([[float(z) for z in x[2:]] for x in splitlines])\n",
    "\n",
    "  nd = len(image_ids)\n",
    "  tp = np.zeros(nd)\n",
    "  fp = np.zeros(nd)\n",
    "\n",
    "  if BB.shape[0] > 0:\n",
    "    # sort by confidence\n",
    "    sorted_ind = np.argsort(-confidence)\n",
    "#    sorted_scores = np.sort(-confidence)\n",
    "    BB = BB[sorted_ind, :]\n",
    "    image_ids = [image_ids[x] for x in sorted_ind]\n",
    "\n",
    "    # go down dets and mark TPs and FPs\n",
    "    for d in range(nd):\n",
    "      id = image_ids[d][-10:-4]\n",
    "      #print(id)\n",
    "      # catch bad detections\n",
    "      try:\n",
    "          R = class_recs[id]\n",
    "      except:\n",
    "        #print(\"det not found\")\n",
    "        continue\n",
    "        \n",
    "      bb = BB[d, :].astype(float)\n",
    "      ovmax = -np.inf\n",
    "      BBGT = R['bbox'].astype(float)\n",
    "\n",
    "      if BBGT.size > 0:\n",
    "        # compute overlaps\n",
    "        # intersection\n",
    "        ixmin = np.maximum(BBGT[:, 0], bb[0])\n",
    "        iymin = np.maximum(BBGT[:, 1], bb[1])\n",
    "        ixmax = np.minimum(BBGT[:, 2], bb[2])\n",
    "        iymax = np.minimum(BBGT[:, 3], bb[3])\n",
    "        iw = np.maximum(ixmax - ixmin + 1., 0.)\n",
    "        ih = np.maximum(iymax - iymin + 1., 0.)\n",
    "        inters = iw * ih\n",
    "\n",
    "        # union\n",
    "        uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +\n",
    "               (BBGT[:, 2] - BBGT[:, 0] + 1.) *\n",
    "               (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)\n",
    "\n",
    "        overlaps = inters / uni\n",
    "        ovmax = np.max(overlaps)\n",
    "        jmax = np.argmax(overlaps)\n",
    "\n",
    "      if ovmax > ovthresh:\n",
    "        if not R['difficult'][jmax]:\n",
    "          if not R['det'][jmax]:\n",
    "            tp[d] = 1.\n",
    "            R['det'][jmax] = 1\n",
    "          else:\n",
    "            fp[d] = 1.\n",
    "      else:\n",
    "        fp[d] = 1.\n",
    "\n",
    "  # compute precision recall\n",
    "  fp = np.cumsum(fp)\n",
    "  tp = np.cumsum(tp)\n",
    "  rec = tp / float(npos)\n",
    "  # avoid divide by zero in case the first detection matches a difficult\n",
    "  # ground truth\n",
    "  prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)\n",
    "  ap = voc_ap(rec, prec, use_07_metric)\n",
    "\n",
    "  return rec, prec, ap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "pascal_classes = np.asarray(['aeroplane', 'bicycle', 'bird', 'boat',\n",
    "               'bottle', 'bus', 'car', 'cat', 'chair',\n",
    "               'cow', 'diningtable', 'dog', 'horse',\n",
    "               'motorbike', 'person', 'pottedplant',\n",
    "               'sheep', 'sofa', 'train', 'tvmonitor'])\n",
    "PASCAL_CLASSES = pascal_classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# write out detections for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "def get_voc_results_file_template(cls):\n",
    "        # VOCdevkit/results/VOC2007/Main/<comp_id>_det_test_aeroplane.txt\n",
    "        filename = 'det_' + \"val\" + '_'+cls+'.txt'\n",
    "        filedir = os.path.join(DEVKIT_PATH, 'results', 'VOC2007', 'Main')\n",
    "        if not os.path.exists(filedir):\n",
    "            os.makedirs(filedir)\n",
    "        path = os.path.join(filedir, filename)\n",
    "        return path\n",
    "\n",
    "\n",
    "def write_voc_results_file(pascal_classes, all_boxes, image_index):\n",
    "        for cls_ind, cls in enumerate(pascal_classes):\n",
    "            if cls == '__background__':\n",
    "                continue\n",
    "            print('Writing {} VOC results file'.format(cls))\n",
    "            filename = get_voc_results_file_template(cls)\n",
    "            with open(filename, 'wt') as f:\n",
    "                for im_ind, index in enumerate(image_index):\n",
    "                    dets = np.asarray(all_boxes[cls_ind][im_ind])\n",
    "                    if dets == []:\n",
    "                        continue\n",
    "                    # the VOCdevkit expects 1-based indices\n",
    "                    for k in range(dets.shape[0]):\n",
    "                        #print(dets[k, 0])\n",
    "                        f.write('{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\\n'.\n",
    "                                format(index, dets[k, 0],\n",
    "                                       dets[k, 1] + 1, dets[k, 2] + 1,\n",
    "                                       dets[k, 3] + 1, dets[k, 4] + 1))\n",
    "import xml.etree.ElementTree as ET\n",
    "def parse_rec(filename):\n",
    "  \"\"\" Parse a PASCAL VOC xml file \"\"\"\n",
    "  tree = ET.parse(filename)\n",
    "  objects = []\n",
    "  for obj in tree.findall('object'):\n",
    "    obj_struct = {}\n",
    "    obj_struct['name'] = obj.find('name').text\n",
    "    obj_struct['pose'] = obj.find('pose').text\n",
    "    obj_struct['truncated'] = int(obj.find('truncated').text)\n",
    "    obj_struct['difficult'] = int(obj.find('difficult').text)\n",
    "    bbox = obj.find('bndbox')\n",
    "    obj_struct['bbox'] = [int(bbox.find('xmin').text),\n",
    "                          int(bbox.find('ymin').text),\n",
    "                          int(bbox.find('xmax').text),\n",
    "                          int(bbox.find('ymax').text)]\n",
    "    objects.append(obj_struct)\n",
    "\n",
    "  return objects\n",
    "def voc_ap(rec, prec, use_07_metric=False):\n",
    "  \"\"\" ap = voc_ap(rec, prec, [use_07_metric])\n",
    "  Compute VOC AP given precision and recall.\n",
    "  If use_07_metric is true, uses the\n",
    "  VOC 07 11 point method (default:False).\n",
    "  \"\"\"\n",
    "  if use_07_metric:\n",
    "    # 11 point metric\n",
    "    ap = 0.\n",
    "    for t in np.arange(0., 1.1, 0.1):\n",
    "      if np.sum(rec >= t) == 0:\n",
    "        p = 0\n",
    "      else:\n",
    "        p = np.max(prec[rec >= t])\n",
    "      ap = ap + p / 11.\n",
    "  else:\n",
    "    # correct AP calculation\n",
    "    # first append sentinel values at the end\n",
    "    mrec = np.concatenate(([0.], rec, [1.]))\n",
    "    mpre = np.concatenate(([0.], prec, [0.]))\n",
    "\n",
    "    # compute the precision envelope\n",
    "    for i in range(mpre.size - 1, 0, -1):\n",
    "      mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n",
    "\n",
    "    # to calculate area under PR curve, look for points\n",
    "    # where X axis (recall) changes value\n",
    "    i = np.where(mrec[1:] != mrec[:-1])[0]\n",
    "\n",
    "    # and sum (\\Delta recall) * prec\n",
    "    ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n",
    "  return ap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm ../VOCdevkit/results/VOC2007/Main/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing aeroplane VOC results file\n",
      "Writing bicycle VOC results file\n",
      "Writing bird VOC results file\n",
      "Writing boat VOC results file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/ipykernel_launcher.py:21: DeprecationWarning: elementwise == comparison failed; this will raise an error in the future.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing bottle VOC results file\n",
      "Writing bus VOC results file\n",
      "Writing car VOC results file\n",
      "Writing cat VOC results file\n",
      "Writing chair VOC results file\n",
      "Writing cow VOC results file\n",
      "Writing diningtable VOC results file\n",
      "Writing dog VOC results file\n",
      "Writing horse VOC results file\n",
      "Writing motorbike VOC results file\n",
      "Writing person VOC results file\n",
      "Writing pottedplant VOC results file\n",
      "Writing sheep VOC results file\n",
      "Writing sofa VOC results file\n",
      "Writing train VOC results file\n",
      "Writing tvmonitor VOC results file\n"
     ]
    }
   ],
   "source": [
    "write_voc_results_file(pascal_classes, all_boxes, val_img_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def python_eval(output_dir='output'):\n",
    "        annopath = os.path.join(\n",
    "            DEVKIT_PATH,\n",
    "            'VOC2007',\n",
    "            'Annotations',\n",
    "            '{:s}.xml')\n",
    "        imagesetfile = os.path.join(\n",
    "            DEVKIT_PATH,\n",
    "            'VOC2007',\n",
    "            'ImageSets',\n",
    "            'Main',\n",
    "            SET + '.txt')\n",
    "        cachedir = os.path.join(DEVKIT_PATH, 'annotations_cache')\n",
    "        aps = []\n",
    "        # The PASCAL VOC metric changed in 2010.\n",
    "        # VOC07 metric is quite old so don't use.\n",
    "        use_07_metric = False\n",
    "        print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))\n",
    "        if not os.path.isdir(output_dir):\n",
    "            os.mkdir(output_dir)\n",
    "        for i, cls in enumerate(PASCAL_CLASSES):\n",
    "            if cls == '__background__':\n",
    "                continue\n",
    "            filename = get_voc_results_file_template(cls)\n",
    "            rec, prec, ap = voc_eval(\n",
    "                filename, annopath, imagesetfile, cls, cachedir, ovthresh=0.5,\n",
    "                use_07_metric=use_07_metric)\n",
    "            aps += [ap]\n",
    "            print('AP for {} = {:.4f}'.format(cls, ap))\n",
    "            with open(os.path.join(output_dir, cls + '_pr.pkl'), 'wb') as f:\n",
    "                pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)\n",
    "        print('Mean AP = {:.4f}'.format(np.mean(aps)))\n",
    "        print('~~~~~~~~')\n",
    "        print('Results:')\n",
    "        for ap in aps:\n",
    "            print('{:.3f}'.format(ap))\n",
    "        print('{:.3f}'.format(np.mean(aps)))\n",
    "        print('~~~~~~~~')\n",
    "        print('')\n",
    "        print('--------------------------------------------------------------')\n",
    "        print('Results computed with the **unofficial** Python eval code.')\n",
    "        print('Results should be very close to the official MATLAB eval code.')\n",
    "        print('Recompute with `./tools/reval.py --matlab ...` for your paper.')\n",
    "        print('-- Thanks, The Management')\n",
    "        print('--------------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VOC07 metric? No\n",
      "AP for aeroplane = 0.8093\n",
      "AP for bicycle = 0.8631\n",
      "AP for bird = 0.7861\n",
      "AP for boat = 0.7077\n",
      "AP for bottle = 0.4031\n",
      "AP for bus = 0.8470\n",
      "AP for car = 0.8519\n",
      "AP for cat = 0.9062\n",
      "AP for chair = 0.5554\n",
      "AP for cow = 0.8457\n",
      "AP for diningtable = 0.7194\n",
      "AP for dog = 0.9009\n",
      "AP for horse = 0.8980\n",
      "AP for motorbike = 0.8618\n",
      "AP for person = 0.7593\n",
      "AP for pottedplant = 0.4591\n",
      "AP for sheep = 0.8089\n",
      "AP for sofa = 0.8020\n",
      "AP for train = 0.8793\n",
      "AP for tvmonitor = 0.7795\n",
      "Mean AP = 0.7722\n",
      "~~~~~~~~\n",
      "Results:\n",
      "0.809\n",
      "0.863\n",
      "0.786\n",
      "0.708\n",
      "0.403\n",
      "0.847\n",
      "0.852\n",
      "0.906\n",
      "0.555\n",
      "0.846\n",
      "0.719\n",
      "0.901\n",
      "0.898\n",
      "0.862\n",
      "0.759\n",
      "0.459\n",
      "0.809\n",
      "0.802\n",
      "0.879\n",
      "0.779\n",
      "0.772\n",
      "~~~~~~~~\n",
      "\n",
      "--------------------------------------------------------------\n",
      "Results computed with the **unofficial** Python eval code.\n",
      "Results should be very close to the official MATLAB eval code.\n",
      "Recompute with `./tools/reval.py --matlab ...` for your paper.\n",
      "-- Thanks, The Management\n",
      "--------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# evaluate detections\n",
    "python_eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}