-
Notifications
You must be signed in to change notification settings - Fork 49
/
evaluation.py
161 lines (142 loc) · 6.58 KB
/
evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
from torch import nn
from tqdm import tqdm
from scipy.ndimage.filters import gaussian_filter
from utils import *
HW = 224 * 224 # image area
n_classes = 1000
def gkern(klen, nsig):
"""Returns a Gaussian kernel array.
Convolution with it results in image blurring."""
# create nxn zeros
inp = np.zeros((klen, klen))
# set element at the middle to one, a dirac delta
inp[klen//2, klen//2] = 1
# gaussian-smooth the dirac, resulting in a gaussian filter mask
k = gaussian_filter(inp, nsig)
kern = np.zeros((3, 3, klen, klen))
kern[0, 0] = k
kern[1, 1] = k
kern[2, 2] = k
return torch.from_numpy(kern.astype('float32'))
def auc(arr):
"""Returns normalized Area Under Curve of the array."""
return (arr.sum() - arr[0] / 2 - arr[-1] / 2) / (arr.shape[0] - 1)
class CausalMetric():
def __init__(self, model, mode, step, substrate_fn):
r"""Create deletion/insertion metric instance.
Args:
model (nn.Module): Black-box model being explained.
mode (str): 'del' or 'ins'.
step (int): number of pixels modified per one iteration.
substrate_fn (func): a mapping from old pixels to new pixels.
"""
assert mode in ['del', 'ins']
self.model = model
self.mode = mode
self.step = step
self.substrate_fn = substrate_fn
def single_run(self, img_tensor, explanation, verbose=0, save_to=None):
r"""Run metric on one image-saliency pair.
Args:
img_tensor (Tensor): normalized image tensor.
explanation (np.ndarray): saliency map.
verbose (int): in [0, 1, 2].
0 - return list of scores.
1 - also plot final step.
2 - also plot every step and print 2 top classes.
save_to (str): directory to save every step plots to.
Return:
scores (nd.array): Array containing scores at every step.
"""
pred = self.model(img_tensor.cuda())
top, c = torch.max(pred, 1)
c = c.cpu().numpy()[0]
n_steps = (HW + self.step - 1) // self.step
if self.mode == 'del':
title = 'Deletion game'
ylabel = 'Pixels deleted'
start = img_tensor.clone()
finish = self.substrate_fn(img_tensor)
elif self.mode == 'ins':
title = 'Insertion game'
ylabel = 'Pixels inserted'
start = self.substrate_fn(img_tensor)
finish = img_tensor.clone()
scores = np.empty(n_steps + 1)
# Coordinates of pixels in order of decreasing saliency
salient_order = np.flip(np.argsort(explanation.reshape(-1, HW), axis=1), axis=-1)
for i in range(n_steps+1):
pred = self.model(start.cuda())
pr, cl = torch.topk(pred, 2)
if verbose == 2:
print('{}: {:.3f}'.format(get_class_name(cl[0][0]), float(pr[0][0])))
print('{}: {:.3f}'.format(get_class_name(cl[0][1]), float(pr[0][1])))
scores[i] = pred[0, c]
# Render image if verbose, if it's the last step or if save is required.
if verbose == 2 or (verbose == 1 and i == n_steps) or save_to:
plt.figure(figsize=(10, 5))
plt.subplot(121)
plt.title('{} {:.1f}%, P={:.4f}'.format(ylabel, 100 * i / n_steps, scores[i]))
plt.axis('off')
tensor_imshow(start[0])
plt.subplot(122)
plt.plot(np.arange(i+1) / n_steps, scores[:i+1])
plt.xlim(-0.1, 1.1)
plt.ylim(0, 1.05)
plt.fill_between(np.arange(i+1) / n_steps, 0, scores[:i+1], alpha=0.4)
plt.title(title)
plt.xlabel(ylabel)
plt.ylabel(get_class_name(c))
if save_to:
plt.savefig(save_to + '/{:03d}.png'.format(i))
plt.close()
else:
plt.show()
if i < n_steps:
coords = salient_order[:, self.step * i:self.step * (i + 1)]
start.cpu().numpy().reshape(1, 3, HW)[0, :, coords] = finish.cpu().numpy().reshape(1, 3, HW)[0, :, coords]
return scores
def evaluate(self, img_batch, exp_batch, batch_size):
r"""Efficiently evaluate big batch of images.
Args:
img_batch (Tensor): batch of images.
exp_batch (np.ndarray): batch of explanations.
batch_size (int): number of images for one small batch.
Returns:
scores (nd.array): Array containing scores at every step for every image.
"""
n_samples = img_batch.shape[0]
predictions = torch.FloatTensor(n_samples, n_classes)
assert n_samples % batch_size == 0
for i in tqdm(range(n_samples // batch_size), desc='Predicting labels'):
preds = self.model(img_batch[i*batch_size:(i+1)*batch_size].cuda()).cpu()
predictions[i*batch_size:(i+1)*batch_size] = preds
top = np.argmax(predictions, -1)
n_steps = (HW + self.step - 1) // self.step
scores = np.empty((n_steps + 1, n_samples))
salient_order = np.flip(np.argsort(exp_batch.reshape(-1, HW), axis=1), axis=-1)
r = np.arange(n_samples).reshape(n_samples, 1)
substrate = torch.zeros_like(img_batch)
for j in tqdm(range(n_samples // batch_size), desc='Substrate'):
substrate[j*batch_size:(j+1)*batch_size] = self.substrate_fn(img_batch[j*batch_size:(j+1)*batch_size])
if self.mode == 'del':
caption = 'Deleting '
start = img_batch.clone()
finish = substrate
elif self.mode == 'ins':
caption = 'Inserting '
start = substrate
finish = img_batch.clone()
# While not all pixels are changed
for i in tqdm(range(n_steps+1), desc=caption + 'pixels'):
# Iterate over batches
for j in range(n_samples // batch_size):
# Compute new scores
preds = self.model(start[j*batch_size:(j+1)*batch_size].cuda())
preds = preds.cpu().numpy()[range(batch_size), top[j*batch_size:(j+1)*batch_size]]
scores[i, j*batch_size:(j+1)*batch_size] = preds
# Change specified number of most salient pixels to substrate pixels
coords = salient_order[:, self.step * i:self.step * (i + 1)]
start.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords] = finish.cpu().numpy().reshape(n_samples, 3, HW)[r, :, coords]
print('AUC: {}'.format(auc(scores.mean(1))))
return scores