-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathpytorchLosses.py
247 lines (189 loc) · 8.04 KB
/
pytorchLosses.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
class sigmoidF1(nn.Module):
def __init__(self, S = -1, E = 0):
super(sigmoidF1, self).__init__()
self.S = S
self.E = E
@torch.cuda.amp.autocast()
def forward(self, y_hat, y):
y_hat = torch.sigmoid(y_hat)
b = torch.tensor(self.S)
c = torch.tensor(self.E)
sig = 1 / (1 + torch.exp(b * (y_hat + c)))
tp = torch.sum(sig * y, dim=0)
fp = torch.sum(sig * (1 - y), dim=0)
fn = torch.sum((1 - sig) * y, dim=0)
sigmoid_f1 = 2*tp / (2*tp + fn + fp + 1e-16)
cost = 1 - sigmoid_f1
macroCost = torch.mean(cost)
return macroCost
class macroSoftF1(nn.Module):
def __init__(self):
super(macroSoftF1, self).__init__()
@torch.cuda.amp.autocast()
def forward(self, y_hat, y):
y_hat = torch.sigmoid(y_hat)
tp = torch.sum(y_hat * y, dim=0)
fp = torch.sum(y_hat * (1 - y), dim=0)
fn = torch.sum((1 - y_hat) * y, dim=0)
macroSoft_f1 = 2*tp / (2*tp + fn + fp + 1e-16)
cost = 1 - macroSoft_f1
macroCost = torch.mean(cost)
return macroCost
# https://github.com/Alibaba-MIIL/ImageNet21K/blob/main/src_files/loss_functions/losses.py
class CrossEntropyLS(nn.Module):
def __init__(self, eps: float = 0.2):
super(CrossEntropyLS, self).__init__()
self.eps = eps
self.logsoftmax = nn.LogSoftmax(dim=-1)
@torch.cuda.amp.autocast()
def forward(self, inputs, target):
num_classes = inputs.size()[-1]
log_preds = self.logsoftmax(inputs)
targets_classes = torch.zeros_like(inputs).scatter_(1, target.long().unsqueeze(1), 1)
targets_classes.mul_(1 - self.eps).add_(self.eps / num_classes)
cross_entropy_loss_tot = -targets_classes.mul(log_preds)
cross_entropy_loss = cross_entropy_loss_tot.sum(dim=-1).mean()
return cross_entropy_loss
# translation from https://github.com/tensorflow/addons/blob/v0.14.0/tensorflow_addons/losses/focal_loss.py#L26-L81
class focalLoss(nn.Module):
def __init__(self, gamma=2.0, alpha=0.25):
super(focalLoss, self).__init__()
self.gamma = gamma
self.alpha = alpha
def forward(self, y_hat, y):
if self.gamma and self.gamma < 0:
raise ValueError("Value of gamma should be greater than or equal to zero.")
ceLoss = torch.nn.BCEWithLogitsLoss()
ce = ceLoss(y, y_hat)
y_hat = torch.sigmoid(y_hat)
p_t = (y * y_hat) + ((1 - y) * (1 - y_hat))
alpha_factor = y * self.alpha + (1 - y) * (1 - self.alpha)
modulating_factor = torch.pow((1.0 - p_t), self.gamma)
focal_loss = torch.sum(alpha_factor * modulating_factor * ce)
return focal_loss
class AsymmetricLoss(nn.Module):
def __init__(self, gamma_neg=4, gamma_pos=1, clip=0.05, eps=1e-8, disable_torch_grad_focal_loss=True):
super(AsymmetricLoss, self).__init__()
self.gamma_neg = gamma_neg
self.gamma_pos = gamma_pos
self.clip = clip
self.disable_torch_grad_focal_loss = disable_torch_grad_focal_loss
self.eps = eps
@torch.cuda.amp.autocast()
def forward(self, x, y):
""""
Parameters
----------
x: input logits
y: targets (multi-label binarized vector)
"""
# Calculating Probabilities
x_sigmoid = torch.sigmoid(x)
xs_pos = x_sigmoid
xs_neg = 1 - x_sigmoid
# Asymmetric Clipping
if self.clip is not None and self.clip > 0:
xs_neg = (xs_neg + self.clip).clamp(max=1)
# Basic CE calculation
los_pos = y * torch.log(xs_pos.clamp(min=self.eps))
los_neg = (1 - y) * torch.log(xs_neg.clamp(min=self.eps))
loss = los_pos + los_neg
# Asymmetric Focusing
if self.gamma_neg > 0 or self.gamma_pos > 0:
if self.disable_torch_grad_focal_loss:
torch.set_grad_enabled(False)
pt0 = xs_pos * y
pt1 = xs_neg * (1 - y) # pt = p if t > 0 else 1-p
pt = pt0 + pt1
one_sided_gamma = self.gamma_pos * y + self.gamma_neg * (1 - y)
one_sided_w = torch.pow(1 - pt, one_sided_gamma)
if self.disable_torch_grad_focal_loss:
torch.set_grad_enabled(True)
loss *= one_sided_w
return -loss.sum()
class AsymmetricLossOptimized(nn.Module):
''' Notice - optimized version, minimizes memory allocation and gpu uploading,
favors inplace operations'''
def __init__(self, gamma_neg=4, gamma_pos=1, clip=0.05, eps=1e-8, disable_torch_grad_focal_loss=False):
super(AsymmetricLossOptimized, self).__init__()
self.gamma_neg = gamma_neg
self.gamma_pos = gamma_pos
self.clip = clip
self.disable_torch_grad_focal_loss = disable_torch_grad_focal_loss
self.eps = eps
# prevent memory allocation and gpu uploading every iteration, and encourages inplace operations
self.targets = self.anti_targets = self.xs_pos = self.xs_neg = self.asymmetric_w = self.loss = None
@torch.cuda.amp.autocast()
def forward(self, x, y):
""""
Parameters
----------
x: input logits
y: targets (multi-label binarized vector)
"""
self.targets = y
self.anti_targets = 1 - y
# Calculating Probabilities
self.xs_pos = torch.sigmoid(x)
self.xs_neg = 1.0 - self.xs_pos
# Asymmetric Clipping
if self.clip is not None and self.clip > 0:
self.xs_neg.add_(self.clip).clamp_(max=1)
# Basic CE calculation
self.loss = self.targets * torch.log(self.xs_pos.clamp(min=self.eps))
self.loss.add_(self.anti_targets * torch.log(self.xs_neg.clamp(min=self.eps)))
# Asymmetric Focusing
if self.gamma_neg > 0 or self.gamma_pos > 0:
if self.disable_torch_grad_focal_loss:
torch.set_grad_enabled(False)
self.xs_pos = self.xs_pos * self.targets
self.xs_neg = self.xs_neg * self.anti_targets
self.asymmetric_w = torch.pow(1 - self.xs_pos - self.xs_neg,
self.gamma_pos * self.targets + self.gamma_neg * self.anti_targets)
if self.disable_torch_grad_focal_loss:
torch.set_grad_enabled(True)
self.loss *= self.asymmetric_w
return -self.loss.sum()
class ASLSingleLabel(nn.Module):
'''
This loss is intended for single-label classification problems
'''
def __init__(self, gamma_pos=0, gamma_neg=4, eps: float = 0.1, reduction='mean'):
super(ASLSingleLabel, self).__init__()
self.eps = eps
self.logsoftmax = nn.LogSoftmax(dim=-1)
self.targets_classes = []
self.gamma_pos = gamma_pos
self.gamma_neg = gamma_neg
self.reduction = reduction
@torch.cuda.amp.autocast()
def forward(self, inputs, target):
'''
"input" dimensions: - (batch_size,number_classes)
"target" dimensions: - (batch_size)
'''
num_classes = inputs.size()[-1]
log_preds = self.logsoftmax(inputs)
self.targets_classes = torch.zeros_like(inputs).scatter_(1, target.long().unsqueeze(1), 1)
# ASL weights
targets = self.targets_classes
anti_targets = 1 - targets
xs_pos = torch.exp(log_preds)
xs_neg = 1 - xs_pos
xs_pos = xs_pos * targets
xs_neg = xs_neg * anti_targets
asymmetric_w = torch.pow(1 - xs_pos - xs_neg,
self.gamma_pos * targets + self.gamma_neg * anti_targets)
log_preds = log_preds * asymmetric_w
if self.eps > 0: # label smoothing
self.targets_classes = self.targets_classes.mul(1 - self.eps).add(self.eps / num_classes)
# loss calculation
loss = - self.targets_classes.mul(log_preds)
loss = loss.sum(dim=-1)
if self.reduction == 'mean':
loss = loss.mean()
return loss