evaluation.py

"""
Copyright (c) 2020-present NAVER Corp.

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""

import argparse
import cv2
import numpy as np
import os
from os.path import join as ospj
import torch.utils.data as torchdata

from config import str2bool
from data_loaders import configure_metadata
from data_loaders import get_image_ids
from data_loaders import get_bounding_boxes
from data_loaders import get_image_sizes
from data_loaders import get_mask_paths
from util import check_scoremap_validity
from util import check_box_convention
from util import t2n

_IMAGENET_MEAN = [0.485, .456, .406]
_IMAGENET_STDDEV = [.229, .224, .225]
_RESIZE_LENGTH = 224
_CONTOUR_INDEX = 1 if cv2.__version__.split('.')[0] == '3' else 0


def calculate_multiple_iou(box_a, box_b):
    """
    Args:
        box_a: numpy.ndarray(dtype=np.int, shape=(num_a, 4))
            x0y0x1y1 convention.
        box_b: numpy.ndarray(dtype=np.int, shape=(num_b, 4))
            x0y0x1y1 convention.
    Returns:
        ious: numpy.ndarray(dtype=np.int, shape(num_a, num_b))
    """
    num_a = box_a.shape[0]
    num_b = box_b.shape[0]

    check_box_convention(box_a, 'x0y0x1y1')
    check_box_convention(box_b, 'x0y0x1y1')

    # num_a x 4 -> num_a x num_b x 4
    box_a = np.tile(box_a, num_b)
    box_a = np.expand_dims(box_a, axis=1).reshape((num_a, num_b, -1))

    # num_b x 4 -> num_b x num_a x 4
    box_b = np.tile(box_b, num_a)
    box_b = np.expand_dims(box_b, axis=1).reshape((num_b, num_a, -1))

    # num_b x num_a x 4 -> num_a x num_b x 4
    box_b = np.transpose(box_b, (1, 0, 2))

    # num_a x num_b
    min_x = np.maximum(box_a[:, :, 0], box_b[:, :, 0])
    min_y = np.maximum(box_a[:, :, 1], box_b[:, :, 1])
    max_x = np.minimum(box_a[:, :, 2], box_b[:, :, 2])
    max_y = np.minimum(box_a[:, :, 3], box_b[:, :, 3])

    # num_a x num_b
    area_intersect = (np.maximum(0, max_x - min_x + 1)
                      * np.maximum(0, max_y - min_y + 1))
    area_a = ((box_a[:, :, 2] - box_a[:, :, 0] + 1) *
              (box_a[:, :, 3] - box_a[:, :, 1] + 1))
    area_b = ((box_b[:, :, 2] - box_b[:, :, 0] + 1) *
              (box_b[:, :, 3] - box_b[:, :, 1] + 1))

    denominator = area_a + area_b - area_intersect
    degenerate_indices = np.where(denominator <= 0)
    denominator[degenerate_indices] = 1

    ious = area_intersect / denominator
    ious[degenerate_indices] = 0
    return ious


def resize_bbox(box, image_size, resize_size):
    """
    Args:
        box: iterable (ints) of length 4 (x0, y0, x1, y1)
        image_size: iterable (ints) of length 2 (width, height)
        resize_size: iterable (ints) of length 2 (width, height)

    Returns:
         new_box: iterable (ints) of length 4 (x0, y0, x1, y1)
    """
    check_box_convention(np.array(box), 'x0y0x1y1')
    box_x0, box_y0, box_x1, box_y1 = map(float, box)
    image_w, image_h = map(float, image_size)
    new_image_w, new_image_h = map(float, resize_size)

    newbox_x0 = box_x0 * new_image_w / image_w
    newbox_y0 = box_y0 * new_image_h / image_h
    newbox_x1 = box_x1 * new_image_w / image_w
    newbox_y1 = box_y1 * new_image_h / image_h
    return int(newbox_x0), int(newbox_y0), int(newbox_x1), int(newbox_y1)


def compute_bboxes_from_scoremaps(scoremap, scoremap_threshold_list,
                                  multi_contour_eval=False):
    """
    Args:
        scoremap: numpy.ndarray(dtype=np.float32, size=(H, W)) between 0 and 1
        scoremap_threshold_list: iterable
        multi_contour_eval: flag for multi-contour evaluation

    Returns:
        estimated_boxes_at_each_thr: list of estimated boxes (list of np.array)
            at each cam threshold
        number_of_box_list: list of the number of boxes at each cam threshold
    """
    check_scoremap_validity(scoremap)
    height, width = scoremap.shape
    scoremap_image = np.expand_dims((scoremap * 255).astype(np.uint8), 2)

    def scoremap2bbox(threshold):
        _, thr_gray_heatmap = cv2.threshold(
            src=scoremap_image,
            thresh=int(threshold * np.max(scoremap_image)),
            maxval=255,
            type=cv2.THRESH_BINARY)
        contours = cv2.findContours(
            image=thr_gray_heatmap,
            mode=cv2.RETR_TREE,
            method=cv2.CHAIN_APPROX_SIMPLE)[_CONTOUR_INDEX]

        if len(contours) == 0:
            return np.asarray([[0, 0, 0, 0]]), 1

        if not multi_contour_eval:
            contours = [max(contours, key=cv2.contourArea)]

        estimated_boxes = []
        for contour in contours:
            x, y, w, h = cv2.boundingRect(contour)
            x0, y0, x1, y1 = x, y, x + w, y + h
            x1 = min(x1, width - 1)
            y1 = min(y1, height - 1)
            estimated_boxes.append([x0, y0, x1, y1])

        return np.asarray(estimated_boxes), len(estimated_boxes)

    estimated_boxes_at_each_thr = []
    number_of_box_list = []
    for threshold in scoremap_threshold_list:
        boxes, number_of_box = scoremap2bbox(threshold)
        estimated_boxes_at_each_thr.append(boxes)
        number_of_box_list.append(number_of_box)

    return estimated_boxes_at_each_thr, number_of_box_list


def calculate_box_size_ratio(object_coordinate, image_size):
    image_w = image_size[0]
    image_h = image_size[1]
    object_w = object_coordinate[2] - object_coordinate[0]
    object_h = object_coordinate[3] - object_coordinate[1]
    size_ratio = object_w * object_h / (image_w * image_h)
    return size_ratio


def load_box_size_ratio(image_ids, original_bboxes, original_images_size):
    object_size_ratios = {}
    for image_id in image_ids:
        list_size_ratio = []
        for original_bbox in original_bboxes[image_id]:
            list_size_ratio.append(calculate_box_size_ratio(original_bbox,
                                                            original_images_size[image_id]))
        object_size_ratios[image_id] = list_size_ratio
    return object_size_ratios


class CamDataset(torchdata.Dataset):
    def __init__(self, scoremap_path, image_ids, norm_method="minmax", percentile=0.6):
        self.scoremap_path = scoremap_path
        self.image_ids = image_ids
        self.norm_method = norm_method
        self.percentile = percentile

    def _load_cam(self, image_id):
        scoremap_file = os.path.join(self.scoremap_path, image_id + '.npy')
        return np.load(scoremap_file)

    def __getitem__(self, index):
        image_id = self.image_ids[index]
        cam = self._load_cam(image_id)
        cam = self.normalize_scoremap(cam)
        return cam, image_id

    def __len__(self):
        return len(self.image_ids)

    def normalize_scoremap(self, cam):
        """
        Args:
            cam: numpy.ndarray(size=(H, W), dtype=np.float)
        Returns:
            numpy.ndarray(size=(H, W), dtype=np.float) between 0 and 1.
            If input array is constant, a zero-array is returned.
        """
        if np.isnan(cam).any():
            return np.zeros_like(cam)
        if cam.min() == cam.max():
            return np.zeros_like(cam)
        if self.norm_method == 'minmax':
            cam -= cam.min()
            cam /= cam.max()
        elif self.norm_method == 'max':
            cam = np.maximum(0, cam)
            cam /= cam.max()
        elif self.norm_method == 'pas':
            cam -= cam.min()
            cam_copy = cam.flatten()
            cam_copy.sort()
            maxx = cam_copy[int(cam_copy.size * 0.9)]
            cam /= maxx
            cam = np.minimum(1, cam)
        elif self.norm_method == 'ivr':
            cam_copy = cam.flatten()
            cam_copy.sort()
            minn = cam_copy[int(cam_copy.size * self.percentile)]
            cam -= minn
            cam = np.maximum(0, cam)
            cam /= cam.max()
        else:
            print('Norm not defined')
        return cam


class LocalizationEvaluator(object):
    """ Abstract class for localization evaluation over score maps.

    The class is designed to operate in a for loop (e.g. batch-wise cam
    score map computation). At initialization, __init__ registers paths to
    annotations and data containers for evaluation. At each iteration,
    each score map is passed to the accumulate() method along with its image_id.
    After the for loop is finalized, compute() is called to compute the final
    localization performance.
    """

    def __init__(self, metadata, dataset_name, split, cam_threshold_list, iou_threshold_list, mask_root,
                 cam_curve_interval, multi_contour_eval, box_size_threshold_list, eval_size_ratio, eval_max_size,
                 compare_box_size, top1_loc):
        self.metadata = metadata
        self.cam_threshold_list = cam_threshold_list
        self.iou_threshold_list = iou_threshold_list
        self.dataset_name = dataset_name
        self.split = split
        self.mask_root = mask_root
        self.cam_curve_interval = cam_curve_interval
        self.multi_contour_eval = multi_contour_eval
        self.eval_size_ratio = eval_size_ratio
        self.box_size_threshold_list = box_size_threshold_list
        self.eval_max_size = eval_max_size
        self.compare_box_size = compare_box_size
        self.top1_loc = top1_loc

    def accumulate(self, scoremap, image_id, top1_cls):
        raise NotImplementedError

    def compute(self):
        raise NotImplementedError


class BoxEvaluator(LocalizationEvaluator):
    def __init__(self, **kwargs):
        super(BoxEvaluator, self).__init__(**kwargs)
        self.image_ids = get_image_ids(metadata=self.metadata)
        self.resize_length = _RESIZE_LENGTH
        self.original_bboxes = get_bounding_boxes(self.metadata)
        self.image_sizes = get_image_sizes(self.metadata)
        self.gt_bboxes = self._load_resized_boxes(self.original_bboxes)
        if self.eval_size_ratio:
            self.cnt = {box_size_threshold: np.zeros(int(1/self.cam_curve_interval))
                        for box_size_threshold in self.box_size_threshold_list}
            self.num_correct = \
                {box_size_threshold: {iou_threshold: np.zeros(len(self.cam_threshold_list))
                                      for iou_threshold in self.iou_threshold_list}
                 for box_size_threshold in self.box_size_threshold_list[:-1]}
            self.object_size_ratios = load_box_size_ratio(self.image_ids,
                                                          self.original_bboxes,
                                                          self.image_sizes)
            self.compare_box_size_matrix = {box_size_threshold: np.zeros(int(1 / self.cam_curve_interval))
                        for box_size_threshold in self.box_size_threshold_list}

            self.eval_point_cam = {box_size_threshold: 0
                                   for box_size_threshold in self.box_size_threshold_list}
            self.cnt_point_cam = {box_size_threshold: 0
                                  for box_size_threshold in self.box_size_threshold_list}
        else:
            self.cnt = 0
            self.num_correct = {iou_threshold: np.zeros(len(self.cam_threshold_list))
                                for iou_threshold in self.iou_threshold_list}
            if self.top1_loc:
                self.num_correct_top1 = \
                                    {iou_threshold: np.zeros(len(self.cam_threshold_list))
                                     for iou_threshold in self.iou_threshold_list}

    def _load_resized_boxes(self, original_bboxes):
        resized_bbox = {image_id: [
            resize_bbox(bbox, self.image_sizes[image_id],
                        (self.resize_length, self.resize_length))
            for bbox in original_bboxes[image_id]]
            for image_id in self.image_ids}
        return resized_bbox

    def accumulate(self, scoremap, image_id, top1_cls=None):
        """
        From a score map, a box is inferred (compute_bboxes_from_scoremaps).
        The box is compared against GT boxes. Count a scoremap as a correct
        prediction if the IOU against at least one box is greater than a certain
        threshold (_IOU_THRESHOLD).

        Args:
            scoremap: numpy.ndarray(size=(H, W), dtype=np.float)
            image_id: string.
        """

        boxes_at_thresholds, number_of_box_list = compute_bboxes_from_scoremaps(
            scoremap=scoremap,
            scoremap_threshold_list=self.cam_threshold_list,
            multi_contour_eval=self.multi_contour_eval)

        boxes_at_thresholds = np.concatenate(boxes_at_thresholds, axis=0)

        multiple_iou = calculate_multiple_iou(
            np.array(boxes_at_thresholds),
            np.array(self.gt_bboxes[image_id]))

        idx = 0
        sliced_multiple_iou = []
        box_size_multiple_iou = []
        compare_box_size = []

        multiple_iou_max = multiple_iou.max(1)
        multiple_iou_index = multiple_iou.argmax(1)
        for nr_box in number_of_box_list:
            sliced_multiple_iou.append(
                max(multiple_iou_max[idx:idx + nr_box]))
            if self.eval_size_ratio:
                if self.eval_max_size:
                    box_size_ratio = max(self.object_size_ratios[image_id])
                else:
                    bbox_max_index = multiple_iou_max[idx:idx + nr_box].argmax()
                    gt_max_index = multiple_iou_index[idx + bbox_max_index]
                    box_size_ratio = self.object_size_ratios[image_id][gt_max_index]
                    if self.compare_box_size:
                        box_size_ratio_eval = calculate_box_size_ratio(boxes_at_thresholds[idx + bbox_max_index],
                                                                  (_RESIZE_LENGTH, _RESIZE_LENGTH))
                        if box_size_ratio != 0:
                            compare_box_size.append(box_size_ratio_eval/box_size_ratio)

                box_size_multiple_iou.append(box_size_ratio)
            idx += nr_box

        if self.eval_size_ratio:
            box_size_multiple_iou = np.array(box_size_multiple_iou)
            # For Point CAM
            box_size_ratio = sum(self.object_size_ratios[image_id]) / len(self.object_size_ratios[image_id])

            if self.compare_box_size:
                compare_box_size = np.array(compare_box_size)
            for _SIZE_PREV, _SIZE_AFTER in zip(self.box_size_threshold_list[:-1], self.box_size_threshold_list[1:]):

                # For Point CAM
                if _SIZE_PREV / 100 < box_size_ratio <= _SIZE_AFTER / 100:
                    self.cnt_point_cam[_SIZE_PREV] += 1
                    for box in self.gt_bboxes[image_id]:
                        x0, y0, x1, y1 = box
                        if x0 * y0 <= scoremap.argmax() <= x1 * y1:
                            self.eval_point_cam[_SIZE_PREV] += 1
                            break

                if _SIZE_AFTER < 100:
                    size_threshold_indices = np.where(np.logical_and(_SIZE_PREV / 100 <= box_size_multiple_iou,
                                                                     box_size_multiple_iou < _SIZE_AFTER / 100))
                else:
                    size_threshold_indices = np.where(np.logical_and(_SIZE_PREV / 100 <= box_size_multiple_iou,
                                                                     box_size_multiple_iou <= _SIZE_AFTER / 100))
                if len(size_threshold_indices[0]) > 0:
                    for _THRESHOLD_IOU in self.iou_threshold_list:
                        correct_threshold_indices = \
                            np.where(np.asarray(sliced_multiple_iou) >= (_THRESHOLD_IOU / 100))[0]

                        correct_threshold_indices = np.intersect1d(correct_threshold_indices, size_threshold_indices)
                        self.num_correct[_SIZE_PREV][_THRESHOLD_IOU][correct_threshold_indices] += 1
                    self.cnt[_SIZE_PREV][size_threshold_indices] += 1
                    if self.compare_box_size:
                        self.compare_box_size_matrix[_SIZE_PREV][size_threshold_indices] \
                            += compare_box_size[size_threshold_indices]
        else:
            for _THRESHOLD_IOU in self.iou_threshold_list:
                correct_threshold_indices = \
                    np.where(np.asarray(sliced_multiple_iou) >= (_THRESHOLD_IOU / 100))[0]
                self.num_correct[_THRESHOLD_IOU][correct_threshold_indices] += 1
                if top1_cls and self.top1_loc:
                    self.num_correct_top1[_THRESHOLD_IOU][correct_threshold_indices] += 1

            self.cnt += 1

    def compute(self):
        """
        Returns:
            max_localization_accuracy: float. The ratio of images where the
               box prediction is correct. The best scoremap threshold is taken
               for the final performance.
        """
        if self.eval_size_ratio:
            # Calculate Total Localization Score
            count_all = np.zeros(len(self.cam_threshold_list))
            correct_all = {iou_threshold: np.zeros(len(self.cam_threshold_list))
                                      for iou_threshold in self.iou_threshold_list}

            for _THRESHOLD_SIZE in self.box_size_threshold_list[:-1]:
                count_all += self.cnt[_THRESHOLD_SIZE]
                for _THRESHOLD_IOU in self.iou_threshold_list:
                    correct_all[_THRESHOLD_IOU] += self.num_correct[_THRESHOLD_SIZE][_THRESHOLD_IOU]

            # For Point CAM
            point_count_all = 0
            point_eval_all = 0
            for _THRESHOLD_SIZE in self.box_size_threshold_list[:-1]:
                point_count_all += self.cnt_point_cam[_THRESHOLD_SIZE]
                point_eval_all += self.eval_point_cam[_THRESHOLD_SIZE]
                if self.cnt_point_cam[_THRESHOLD_SIZE] != 0:
                    self.eval_point_cam[_THRESHOLD_SIZE] /= self.cnt_point_cam[_THRESHOLD_SIZE]

            max_box_acc = {box_size_threshold: [] for box_size_threshold in self.box_size_threshold_list[:-1]}
            max_box_acc_total = []

            max_box_acc_best_threshold = {box_size_threshold: [] for box_size_threshold in self.box_size_threshold_list[:-1]}
            print_count = {box_size_threshold: 0 for box_size_threshold in self.box_size_threshold_list[:-1]}
            compare_count = {box_size_threshold: 0 for box_size_threshold in self.box_size_threshold_list[:-1]}
            for _THRESHOLD_IOU in self.iou_threshold_list:
                localization_accuracies = correct_all[_THRESHOLD_IOU] * 100. / count_all
                best_threshold = localization_accuracies.argmax()
                for _THRESHOLD_SIZE in self.box_size_threshold_list[:-1]:
                    if np.all(self.cnt[_THRESHOLD_SIZE] != 0):
                        localization_find_threshold = self.num_correct[_THRESHOLD_SIZE][_THRESHOLD_IOU] * 100. / \
                                                      self.cnt[_THRESHOLD_SIZE]
                        max_box_acc_best_threshold[_THRESHOLD_SIZE] = self.cam_curve_interval * \
                                                                      localization_find_threshold.argmax()

                        localization_size = self.num_correct[_THRESHOLD_SIZE][_THRESHOLD_IOU][best_threshold] * 100. / \
                                                      self.cnt[_THRESHOLD_SIZE][best_threshold]
                        max_box_acc[_THRESHOLD_SIZE].append(localization_size)
                    else:
                        max_box_acc[_THRESHOLD_SIZE].append(0)
                    print_count[_THRESHOLD_SIZE] = self.cnt[_THRESHOLD_SIZE][best_threshold]
                    if self.compare_box_size:
                        compare_count[_THRESHOLD_SIZE] = self.compare_box_size_matrix[_THRESHOLD_SIZE][best_threshold] \
                                                         / print_count[_THRESHOLD_SIZE]

                max_box_acc_total.append(localization_accuracies.max())

            if self.compare_box_size:
                print("Compared Box Size :", compare_count.values())
            # If you want to inspect the other experiments, remove the annotation head of the line.
            # print("Point CAM :", self.eval_point_cam.values())
            # print("Point CAM Total :", point_eval_all/point_count_all)
            # print("Threshold List :", max_box_acc_best_threshold.values())
            # print("Best Threshold :", best_threshold)
            # print(print_count)
            print('MaxBoxAccV2 Score :', np.average(max_box_acc_total))
        else:
            max_box_acc = []
            for _THRESHOLD in self.iou_threshold_list:
                if self.top1_loc:
                    localization_accuracies = self.num_correct_top1[_THRESHOLD] * 100. / \
                                              float(self.cnt)
                    max_box_acc.append(localization_accuracies.max())
                else:
                    localization_accuracies = self.num_correct[_THRESHOLD] * 100. / \
                                              float(self.cnt)
                    max_box_acc.append(localization_accuracies.max())

        return max_box_acc


def load_mask_image(file_path, resize_size):
    """
    Args:
        file_path: string.
        resize_size: tuple of ints (height, width)
    Returns:
        mask: numpy.ndarray(dtype=numpy.float32, shape=(height, width))
    """
    mask = np.float32(cv2.imread(file_path, cv2.IMREAD_GRAYSCALE))
    mask = cv2.resize(mask, resize_size, interpolation=cv2.INTER_NEAREST)
    return mask


def get_mask(mask_root, mask_paths, ignore_path):
    """
    Ignore mask is set as the ignore box region \setminus the ground truth
    foreground region.

    Args:
        mask_root: string.
        mask_paths: iterable of strings.
        ignore_path: string.

    Returns:
        mask: numpy.ndarray(size=(224, 224), dtype=np.uint8)
    """
    mask_all_instances = []
    for mask_path in mask_paths:
        mask_file = os.path.join(mask_root, mask_path)
        mask = load_mask_image(mask_file, (_RESIZE_LENGTH, _RESIZE_LENGTH))
        mask_all_instances.append(mask > 0.5)
    mask_all_instances = np.stack(mask_all_instances, axis=0).any(axis=0)

    ignore_file = os.path.join(mask_root, ignore_path)
    ignore_box_mask = load_mask_image(ignore_file,
                                      (_RESIZE_LENGTH, _RESIZE_LENGTH))
    ignore_box_mask = ignore_box_mask > 0.5

    ignore_mask = np.logical_and(ignore_box_mask,
                                 np.logical_not(mask_all_instances))

    if np.logical_and(ignore_mask, mask_all_instances).any():
        raise RuntimeError("Ignore and foreground masks intersect.")

    return (mask_all_instances.astype(np.uint8) +
            255 * ignore_mask.astype(np.uint8))


class MaskEvaluator(LocalizationEvaluator):
    def __init__(self, **kwargs):
        super(MaskEvaluator, self).__init__(**kwargs)

        if self.dataset_name != "OpenImages":
            raise ValueError("Mask evaluation must be performed on OpenImages.")

        self.mask_paths, self.ignore_paths = get_mask_paths(self.metadata)

        # cam_threshold_list is given as [0, bw, 2bw, ..., 1-bw]
        # Set bins as [0, bw), [bw, 2bw), ..., [1-bw, 1), [1, 2), [2, 3)
        self.num_bins = len(self.cam_threshold_list) + 2
        self.threshold_list_right_edge = np.append(self.cam_threshold_list,
                                                   [1.0, 2.0, 3.0])
        if self.eval_size_ratio:
            self.gt_true_score_hist = {box_size_threshold: np.zeros(self.num_bins, dtype=np.float)
                                       for box_size_threshold in self.box_size_threshold_list}
            self.gt_false_score_hist = {box_size_threshold: np.zeros(self.num_bins, dtype=np.float)
                                        for box_size_threshold in self.box_size_threshold_list}
            self.count_size_ratio = {box_size_threshold: 0 for box_size_threshold in self.box_size_threshold_list}
            self.eval_point_cam = {box_size_threshold: 0
                                   for box_size_threshold in self.box_size_threshold_list}
            self.cnt_point_cam = {box_size_threshold: 0
                                  for box_size_threshold in self.box_size_threshold_list}
        else:
            self.gt_true_score_hist = np.zeros(self.num_bins, dtype=np.float)
            self.gt_false_score_hist = np.zeros(self.num_bins, dtype=np.float)

    def accumulate(self, scoremap, image_id, top1_cls=None):
        """
        Score histograms over the score map values at GT positive and negative
        pixels are computed.

        Args:
            scoremap: numpy.ndarray(size=(H, W), dtype=np.float)
            image_id: string.
        """
        check_scoremap_validity(scoremap)
        gt_mask = get_mask(self.mask_root,
                           self.mask_paths[image_id],
                           self.ignore_paths[image_id])

        if self.eval_size_ratio:
            find_index, true_count = np.unique(gt_mask, return_counts=True)
            true_index = np.where(find_index == 1)
            mask_size_ratio = true_count[true_index] / (gt_mask.shape[0] * gt_mask.shape[1])

        gt_true_scores = scoremap[gt_mask == 1]
        gt_false_scores = scoremap[gt_mask == 0]

        if self.eval_size_ratio:
            for _SIZE_PREV, _SIZE_AFTER in zip(self.box_size_threshold_list[:-1], self.box_size_threshold_list[1:]):
                _SIZE_AFTER = 101 if _SIZE_AFTER == 100 else _SIZE_AFTER
                if _SIZE_PREV <= mask_size_ratio * 100 < _SIZE_AFTER:
                    # For Point CAM
                    self.cnt_point_cam[_SIZE_PREV] += 1
                    if np.any(gt_mask[scoremap == 1.0] == 1):
                        self.eval_point_cam[_SIZE_PREV] += 1
                    gt_true_hist, _ = np.histogram(gt_true_scores,
                                                   bins=self.threshold_list_right_edge)
                    self.gt_true_score_hist[_SIZE_PREV] += gt_true_hist.astype(np.float)

                    gt_false_hist, _ = np.histogram(gt_false_scores,
                                                    bins=self.threshold_list_right_edge)
                    self.gt_false_score_hist[_SIZE_PREV] += gt_false_hist.astype(np.float)
                    self.count_size_ratio[_SIZE_PREV] += 1
                    break
        else:
            gt_true_hist, _ = np.histogram(gt_true_scores,
                                           bins=self.threshold_list_right_edge)
            self.gt_true_score_hist += gt_true_hist.astype(np.float)

            gt_false_hist, _ = np.histogram(gt_false_scores,
                                            bins=self.threshold_list_right_edge)
            self.gt_false_score_hist += gt_false_hist.astype(np.float)

    def compute(self):
        """
        Arrays are arranged in the following convention (bin edges):

        gt_true_score_hist: [0.0, eps), ..., [1.0, 2.0), [2.0, 3.0)
        gt_false_score_hist: [0.0, eps), ..., [1.0, 2.0), [2.0, 3.0)
        tp, fn, tn, fp: >=2.0, >=1.0, ..., >=0.0

        Returns:
            auc: float. The area-under-curve of the precision-recall curve.
               Also known as average precision (AP).
        """
        if self.eval_size_ratio:
            # For Evaluate Total Score
            tp_all = np.zeros(self.num_bins, dtype=np.float)
            fn_all = np.zeros(self.num_bins, dtype=np.float)
            fp_all = np.zeros(self.num_bins, dtype=np.float)
            tn_all = np.zeros(self.num_bins, dtype=np.float)

            auc_list = {box_size_threshold: [] for box_size_threshold in self.box_size_threshold_list[:-1]}

            # For Point CAM
            point_count_all = 0
            point_eval_all = 0
            for _THRESHOLD_SIZE in self.box_size_threshold_list[:-1]:
                point_count_all += self.cnt_point_cam[_THRESHOLD_SIZE]
                point_eval_all += self.eval_point_cam[_THRESHOLD_SIZE]

            for i in range(len(self.box_size_threshold_list) - 1):
                num_gt_true = self.gt_true_score_hist[self.box_size_threshold_list[i]].sum()
                tp = self.gt_true_score_hist[self.box_size_threshold_list[i]][::-1].cumsum()
                fn = num_gt_true - tp

                num_gt_false = self.gt_false_score_hist[self.box_size_threshold_list[i]].sum()
                fp = self.gt_false_score_hist[self.box_size_threshold_list[i]][::-1].cumsum()
                tn = num_gt_false - fp

                # For Total Score
                tp_all += tp
                fn_all += fn
                fp_all += fp
                tn_all += tn

                if ((tp + fn) <= 0).all():
                    raise RuntimeError("No positive ground truth in the eval set.")
                if ((tp + fp) <= 0).all():
                    raise RuntimeError("No positive prediction in the eval set.")

                non_zero_indices = (tp + fp) != 0

                precision = tp / (tp + fp)
                recall = tp / (tp + fn)

                auc = (precision[1:] * np.diff(recall))[non_zero_indices[1:]].sum()
                auc *= 100

                auc_list[self.box_size_threshold_list[i]].append(auc)

                # For Point CAM
                self.eval_point_cam[self.box_size_threshold_list[i]] /= self.cnt_point_cam[self.box_size_threshold_list[i]]

            # For Total Score
            non_zero_indices = (tp_all + fp_all) != 0

            precision = tp_all / (tp_all + fp_all)
            recall = tp_all / (tp_all + fn_all)

            auc_total = (precision[1:] * np.diff(recall))[non_zero_indices[1:]].sum()
            auc_total *= 100

            print(self.eval_point_cam, point_eval_all/point_count_all)
            print('Final Localization Score :', auc_total)
            print(self.count_size_ratio)
            return auc_list
        else:
            num_gt_true = self.gt_true_score_hist.sum()
            tp = self.gt_true_score_hist[::-1].cumsum()
            fn = num_gt_true - tp

            num_gt_false = self.gt_false_score_hist.sum()
            fp = self.gt_false_score_hist[::-1].cumsum()
            tn = num_gt_false - fp

            if ((tp + fn) <= 0).all():
                raise RuntimeError("No positive ground truth in the eval set.")
            if ((tp + fp) <= 0).all():
                raise RuntimeError("No positive prediction in the eval set.")

            non_zero_indices = (tp + fp) != 0

            precision = tp / (tp + fp)
            recall = tp / (tp + fn)

            auc = (precision[1:] * np.diff(recall))[non_zero_indices[1:]].sum()
            auc *= 100

            print("Mask AUC on split {}: {}".format(self.split, auc))
            return auc


def _get_cam_loader(image_ids, scoremap_path, norm_method):
    return torchdata.DataLoader(
        CamDataset(scoremap_path, image_ids, norm_method=norm_method),
        batch_size=128,
        shuffle=False,
        num_workers=1,
        pin_memory=True)


def evaluate_wsol(scoremap_root, metadata_root, mask_root, dataset_name, split,
                  multi_contour_eval, multi_iou_eval, iou_threshold_list,
                  box_size_threshold_list, eval_size_ratio, eval_max_size,
                  compare_box_size, top1_loc,
                  norm_method="minmax",
                  cam_curve_interval=.001):
    """
    Compute WSOL performances of predicted heatmaps against ground truth
    boxes (CUB, ILSVRC) or masks (OpenImages). For boxes, we compute the
    gt-known box accuracy (IoU>=0.5) at the optimal heatmap threshold.
    For masks, we compute the area-under-curve of the pixel-wise precision-
    recall curve.

    Args:
        scoremap_root: string. Score maps for each eval image are saved under
            the output_path, with the name corresponding to their image_ids.
            For example, the heatmap for the image "123/456.JPEG" is expected
            to be located at "{output_path}/123/456.npy".
            The heatmaps must be numpy arrays of type np.float, with 2
            dimensions corresponding to height and width. The height and width
            must be identical to those of the original image. The heatmap values
            must be in the [0, 1] range. The map must attain values 0.0 and 1.0.
            See check_scoremap_validity() in util.py for the exact requirements.
        metadata_root: string.
        mask_root: string.
        dataset_name: string. Supports [CUB, ILSVRC, and OpenImages].
        split: string. Supports [train, val, test].
        multi_contour_eval:  considering the best match between the set of all
            estimated boxes and the set of all ground truth boxes.
        multi_iou_eval: averaging the performance across various level of iou
            thresholds.
        iou_threshold_list: list. default: [30, 50, 70]
        cam_curve_interval: float. Default 0.001. At which threshold intervals
            will the heatmaps be evaluated?
    Returns:
        performance: float. For CUB and ILSVRC, maxboxacc is returned.
            For OpenImages, area-under-curve of the precision-recall curve
            is returned.
    """
    print("Loading and evaluating cams.")
    meta_path = os.path.join(metadata_root, dataset_name, split)
    metadata = configure_metadata(meta_path)
    image_ids = get_image_ids(metadata)
    cam_threshold_list = list(np.arange(0, 1, cam_curve_interval))

    evaluator = {"OpenImages": MaskEvaluator,
                 "CUB": BoxEvaluator,
                 "ILSVRC": BoxEvaluator
                 }[dataset_name](metadata=metadata,
                                 dataset_name=dataset_name,
                                 split=split,
                                 cam_threshold_list=cam_threshold_list,
                                 mask_root=ospj(mask_root, 'OpenImages'),
                                 cam_curve_interval=cam_curve_interval,
                                 multi_contour_eval=multi_contour_eval,
                                 iou_threshold_list=iou_threshold_list,
                                 box_size_threshold_list=box_size_threshold_list,
                                 eval_size_ratio=eval_size_ratio,
                                 eval_max_size=eval_max_size,
                                 compare_box_size=compare_box_size,
                                 top1_loc=top1_loc)

    cam_loader = _get_cam_loader(image_ids, scoremap_root, norm_method)
    for cams, image_ids in cam_loader:
        for cam, image_id in zip(cams, image_ids):
            evaluator.accumulate(t2n(cam), image_id)
    performance = evaluator.compute()
    if eval_size_ratio:
        for box_size_threshold in box_size_threshold_list[:-1]:
            if multi_contour_eval or dataset_name == 'OpenImages':
                performance[box_size_threshold] = np.average(performance[box_size_threshold])
            else:
                performance[box_size_threshold] = performance[box_size_threshold][iou_threshold_list.index(50)]

        for _SIZE_PREV, _SIZE_AFTER in zip(box_size_threshold_list[:-1], box_size_threshold_list[1:]):
            print('box size ratio: {}~{}, localization : {}'.format(_SIZE_PREV, _SIZE_AFTER, performance[_SIZE_PREV]))
    else:
        if multi_iou_eval or dataset_name == 'OpenImages':
            performance = np.average(performance)
        print('localization: {}'.format(performance))
    return performance


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--scoremap_root', type=str,
                        default='train_log/scoremaps/',
                        help="The root folder for score maps to be evaluated.")
    parser.add_argument('--metadata_root', type=str, default='metadata/',
                        help="Root folder of metadata.")
    parser.add_argument('--mask_root', type=str, default='dataset/',
                        help="Root folder of masks (OpenImages).")
    parser.add_argument('--dataset_name', type=str,
                        help="One of [CUB, ImageNet, OpenImages].")
    parser.add_argument('--split', type=str,
                        help="One of [val, test]. They correspond to "
                             "train-fullsup and test, respectively.")
    parser.add_argument('--cam_curve_interval', type=float, default=0.001,
                        help="At which threshold intervals will the score maps "
                             "be evaluated?.")
    parser.add_argument('--multi_contour_eval', type=str2bool, nargs='?',
                        const=True, default=False)
    parser.add_argument('--multi_iou_eval', type=str2bool, nargs='?',
                        const=True, default=False)
    parser.add_argument('--iou_threshold_list', nargs='+',
                        type=int, default=[30, 50, 70])
    # Add Small Box threshold list
    parser.add_argument('--box_size_threshold_list', nargs='+', type=int, default=[0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100])
    parser.add_argument('--eval_size_ratio', type=str2bool, nargs='?',
                        const=True, default=False)
    parser.add_argument('--eval_max_size', type=str2bool, nargs='?',
                        const=True, default=False)
    parser.add_argument('--compare_box_size', type=str2bool, nargs='?',
                        const=True, default=False)
    parser.add_argument('--norm_method', type=str)
    parser.add_argument('--top1_loc', type=str2bool, nargs='?',
                        const=True, default=False)

    args = parser.parse_args()
    evaluate_wsol(scoremap_root=args.scoremap_root,
                  metadata_root=args.metadata_root,
                  mask_root=args.mask_root,
                  dataset_name=args.dataset_name,
                  split=args.split,
                  cam_curve_interval=args.cam_curve_interval,
                  multi_contour_eval=args.multi_contour_eval,
                  multi_iou_eval=args.multi_iou_eval,
                  iou_threshold_list=args.iou_threshold_list,
                  box_size_threshold_list=args.box_size_threshold_list,
                  eval_size_ratio=args.eval_size_ratio,
                  eval_max_size=args.eval_max_size,
                  compare_box_size=args.compare_box_size,
                  norm_method=args.norm_method,
                  top1_loc=args.top1_loc)


if __name__ == "__main__":
    main()