process_results.py

from __future__ import print_function
import argparse
import os
import pickle
import sklearn
import numpy as np
import utils


def submodular_anchor_precrecall(z_anchor, dataset, preds_validation,
                                 preds_test, k):
    # returns picked, precisions, recalls
    picked = utils.greedy_pick_anchor(
        z_anchor['exps'],
        dataset.data[z_anchor['validation_idx']],
        k=k,
        threshold=1.1)
    precs = []
    recs = []
    for i in range(1, k + 1):
        exs = picked[:i]
        anchors = [z_anchor['exps'][i] for i in exs]
        data_anchors = dataset.data[z_anchor['validation_idx']][exs]
        pred_anchors = preds_validation[exs]
        prec, rec = utils.evaluate_anchor(
            anchors, data_anchors, pred_anchors,
            dataset.data[z_anchor['test_idx']], preds_test,
            threshold=1.1)
        precs.append(prec)
        recs.append(rec)
    return picked, precs, recs


def random_anchor_precrecall(z_anchor, dataset, preds_validation, preds_test,
                             k, do_all=False):
    # returns (anchor_prec, anchor_rec, anchor_prec_std, anchor_rec_std)
    temp_prec_anchor = []
    temp_rec_anchor = []
    all_ids = range(len(z_anchor['exps']))
    if do_all:
        all_exs = np.array(all_ids).reshape((-1, 1))
    else:
        all_exs = [np.random.choice(all_ids, k, replace=False) for x in range(500)]
    for exs in all_exs:
        anchors = [z_anchor['exps'][i] for i in exs]
        data_anchors = dataset.data[z_anchor['validation_idx']][exs]
        pred_anchors = preds_validation[exs]
        prec, rec = utils.evaluate_anchor(
            anchors, data_anchors, pred_anchors,
            dataset.data[z_anchor['test_idx']], preds_test,
            threshold=1.1)
        n = preds_test.shape[0]
        temp_prec_anchor.append(prec)
        temp_rec_anchor.append(rec)
    # print (predicted_right / predicted, predicted / total, 0, 0)
    return (np.mean(temp_prec_anchor), np.mean(temp_rec_anchor),
            np.std(temp_prec_anchor), np.std(temp_rec_anchor))


def random_until_k(random_fn, k):
    # random_fn takes k and returns prec, rec, prec_std, rec_std
    # returns (precisions, recalls, precisions_stds, recalls_stds)
    precs = []
    recs = []
    prec_stds = []
    rec_stds = []
    for i in range(1, k + 1):
        prec, rec, prec_std, rec_std = random_fn(i)
        precs.append(prec)
        recs.append(rec)
        prec_stds.append(prec_std)
        rec_stds.append(rec_std)
    return precs, recs, prec_stds, rec_stds


def submodular_lime_precrecall(z_lime, dataset, preds_validation, preds_test,
                               k, val_weights, val_vals, desired_precision=0,
                               to_change='distance', verbose=False,
                               threshold=0, pred_threshold=0.5):
    # returns (picked, precisions, recalls, threshold, pred_threshold)
    binary = np.bincount(preds_test).shape[0] <= 2
    if desired_precision != 0:
        thresholds = (np.linspace(0, 1, 101) if to_change == 'distance' else
                      np.linspace(0.5, 1, 51))
        for t in thresholds:
            if to_change == 'distance':
                threshold = t
            elif to_change == 'pred':
                pred_threshold = t
            else:
                print('Error: to_change must be pred or distance, it is',
                      to_change)
                quit()
            picked = utils.submodular_coverage_pick(val_weights, val_vals,
                                                    threshold, pred_threshold,
                                                    binary, k, verbose=False)
            exps = [z_lime['exps'][i] for i in picked]
            data_exps = dataset.data[z_lime['validation_idx']][picked]
            preds_exps = preds_validation[picked]
            w, v = utils.compute_lime_weight_vals(
                exps, data_exps, dataset.data[z_lime['test_idx']])
            prec, rec = utils.evaluate_lime(
                w, v, preds_exps, preds_test, threshold, pred_threshold,
                binary=binary)
            if verbose:
                print(t, prec, rec)
            if prec >= desired_precision:
                break
    picked_ = utils.submodular_coverage_pick(val_weights, val_vals,
                                             threshold, pred_threshold,
                                             binary, k)
    precs = []
    recs = []
    for i in range(1, k + 1):
        picked = picked_[:i]
        exps = [z_lime['exps'][i] for i in picked]
        data_exps = dataset.data[z_lime['validation_idx']][picked]
        preds_exps = preds_validation[picked]
        weights, vals = utils.compute_lime_weight_vals(
            exps, data_exps, dataset.data[z_lime['test_idx']])
        prec, rec = utils.evaluate_lime(weights, vals, preds_exps, preds_test,
                                        threshold, pred_threshold,
                                        binary=binary)
        precs.append(prec)
        recs.append(rec)
    return picked, precs, recs, threshold, pred_threshold

    return (prec, rec, threshold, pred_threshold)


def random_lime_precrecall(
    z_lime, dataset, preds_validation, preds_test, k, desired_precision=0,
    to_change='distance', do_all=False, verbose=False,
    threshold=0, pred_threshold=0.5):
    binary = np.bincount(preds_test).shape[0] <= 2
    all_ids = range(len(z_lime['exps']))
    if do_all:
        all_exs = np.array(all_ids).reshape((-1, 1))
    else:
        all_exs = [np.random.choice(all_ids, k, replace=False) for x in range(500)]
    ws = []
    vs = []
    for picked in all_exs:
        exps = [z_lime['exps'][i] for i in picked]
        data_exps = dataset.data[z_lime['validation_idx']][picked]
        w, v = utils.compute_lime_weight_vals(
            exps, data_exps, dataset.data[z_lime['test_idx']])
        ws.append(w)
        vs.append(v)

    thresholds = (np.linspace(0, 1, 101) if to_change == 'distance' else
                  np.linspace(0.5, 1, 51))
    for t in thresholds:
        if desired_precision != 0:
            if to_change == 'distance':
                threshold = t
            elif to_change == 'pred':
                pred_threshold = t
            else:
                print('Error: to_change must be pred or distance, it is',
                      to_change)
                quit()
        temp_prec_lime = []
        temp_rec_lime = []

        for picked, w, v in zip(all_exs, ws, vs):
            preds_exps = preds_validation[picked]
            prec, rec = utils.evaluate_lime(
                w, v, preds_exps, preds_test, threshold, pred_threshold,
                binary=binary)
            temp_prec_lime.append(prec)
            temp_rec_lime.append(rec)
        if verbose:
            print(t, np.mean(temp_prec_lime), np.mean(temp_rec_lime))
        if np.mean(temp_prec_lime) >= desired_precision:
            return (np.mean(temp_prec_lime), np.mean(temp_rec_lime),
                    np.std(temp_prec_lime), np.std(temp_rec_lime),
                    threshold, pred_threshold)
    return (1, 0, 0, 0, 1, 1)

def main():
    parser = argparse.ArgumentParser(description='Graphs')
    parser.add_argument(
        '-p', dest='pickle_folder',
        default='./out_pickles')
    parser.add_argument('-d', dest='dataset', required=True,
                        choices=['adult', 'recidivism', 'lending'],
                        help='dataset to use')
    parser.add_argument('-m', dest='model', required=True,
                        choices=['xgboost', 'logistic', 'nn'],
                        help='model: xgboost, logistic or nn')
    parser.add_argument(
        '-o', dest='output_folder',
        default='./results')

    args = parser.parse_args()
    dataset = utils.load_dataset(args.dataset, balance=True)
    dataset_name = args.dataset
    algorithm = args.model
    z_anchor = pickle.load(
        open(os.path.join(args.pickle_folder, '%s-anchor-%s' % (
            dataset_name, algorithm))))
    z_lime = pickle.load(
        open(os.path.join(args.pickle_folder, '%s-lime-%s' % (
            dataset_name, algorithm))))
    preds_validation = z_anchor['model'].predict(
        z_anchor['encoder'].transform(
            dataset.data[z_anchor['validation_idx']]))
    preds_test = z_anchor['model'].predict(
        z_anchor['encoder'].transform(
            dataset.data[z_anchor['test_idx']]))
    ret = {}
    ret['accuracy'] = sklearn.metrics.accuracy_score(
        dataset.labels[z_anchor['test_idx']], preds_test)
    print('accuracy', ret['accuracy'])

    print('Lime weights')
    val_weights, val_vals = utils.compute_lime_weight_vals(
        z_lime['exps'], dataset.data[z_lime['validation_idx']],
        dataset.data[z_lime['validation_idx']])

    print('Submodular anchor')
    picked, precs, recs = submodular_anchor_precrecall(
        z_anchor, dataset, preds_validation, preds_test, 10)
    ret['anchor_submodular'] = (picked, precs, recs)
    anchor_prec = precs[-1]

    print('Submodular lime pred')
    picked, precs, recs, t1, t2 = submodular_lime_precrecall(
        z_lime, dataset, preds_validation, preds_test, 10, val_weights,
        val_vals, desired_precision=anchor_prec, to_change='pred',
        verbose=True)

    ret['lime_pred_submodular'] = (picked, precs, recs)
    ret['lime_pred_submodular_threshold'] = t2

    print('Random anchor')
    (prec, cov, prec_std, cov_std) = random_anchor_precrecall(
        z_anchor, dataset, preds_validation, preds_test, 1, do_all=True)
    ret['anchor_1'] = (prec, cov, prec_std, cov_std)

    print('Random lime')
    (prec, cov, prec_std, cov_std, _, _) = random_lime_precrecall(
        z_lime, dataset, preds_validation, preds_test, k=1,
        desired_precision=0.0, to_change='distance', verbose=True,
        do_all=True)
    ret['lime_naive_1'] = (prec, cov, prec_std, cov_std)

    # print('Distance random lime')
    # (prec, cov, prec_std, cov_std, t1, t2) = random_lime_precrecall(
    #     z_lime, dataset, preds_validation, preds_test, k=1,
    #     desired_precision=0.0, to_change='distance', verbose=True,
    #     do_all=True, threshold=ret['lime_distance_submodular_threshold'])
    # ret['lime_distance_1'] = (prec, cov, prec_std, cov_std)
    # ret['lime_distance_1_threshold'] = t1

    print('Pred random lime')
    (prec, cov, prec_std, cov_std, t1, t2) = random_lime_precrecall(
        z_lime, dataset, preds_validation, preds_test, k=1,
        desired_precision=0.0, to_change='pred', verbose=True,
        do_all=True, pred_threshold=ret['lime_pred_submodular_threshold'])
    ret['lime_pred_1'] = (prec, cov, prec_std, cov_std)
    ret['lime_pred_1_threshold'] = t2

    def random_fn_lime(k):
        return random_lime_precrecall(
            z_lime, dataset, preds_validation, preds_test, k=k,
            desired_precision=0.0, to_change='pred', verbose=True,
            do_all=False,
            pred_threshold=ret['lime_pred_submodular_threshold'])[:4]

    def random_fn_anchor(k):
        return random_anchor_precrecall(
            z_anchor, dataset, preds_validation, preds_test, k, do_all=False)

    ret['anchor_random'] = random_until_k(random_fn_anchor, 10)
    ret['lime_pred_random'] = random_until_k(random_fn_lime, 10)

    path = os.path.join(args.output_folder, '%s-%s.pickle' % (
        dataset_name, algorithm))

    pickle.dump(ret, open(path, 'w'))

if __name__ == '__main__':
    main()