-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathmodels.py
196 lines (171 loc) · 8.06 KB
/
models.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
from torch import nn
from ops.basic_ops import ConsensusModule
from utils.transforms import *
from torch.nn.init import normal_, constant_
from scipy.ndimage import zoom
import cv2
import os, sys
from torch.cuda import amp
class VideoModel(nn.Module):
def __init__(self, num_class, num_segments,
base_model='BNInception',
consensus_type='avg', before_softmax=True,
dropout=0.5, crop_num=1, print_spec=True,
gsf=True, gsf_ch_ratio=100,
target_transform=None):
super(VideoModel, self).__init__()
self.num_segments = num_segments
self.before_softmax = before_softmax
self.dropout = dropout
self.crop_num = crop_num
self.consensus_type = consensus_type
self.gsf = gsf
self.gsf_ch_ratio = gsf_ch_ratio
self.target_transform = target_transform
if not before_softmax and consensus_type != 'avg':
raise ValueError("Only avg consensus can be used after Softmax")
if print_spec:
print(("""
Initializing Video Model with backbone: {}.
Model Configurations:
GSF: {}
Channel ratio: {}
num_segments: {}
consensus_module: {}
dropout_ratio: {}
""".format(base_model, self.gsf, self.gsf_ch_ratio, self.num_segments, consensus_type, self.dropout)))
self.feature_dim = self._prepare_base_model(base_model)
self.feature_dim = self._prepare_model(num_class, self.feature_dim)
self.consensus = ConsensusModule(consensus_type)
if not self.before_softmax:
self.softmax = nn.Softmax()
def _prepare_model(self, num_class, feature_dim):
if self.dropout == 0:
setattr(self.base_model, self.base_model.last_layer_name, nn.Linear(feature_dim, num_class))
self.new_fc = None
else:
setattr(self.base_model, self.base_model.last_layer_name, nn.Dropout(p=self.dropout))
self.new_fc = nn.Linear(feature_dim, num_class)
std = 0.001
if self.new_fc is None:
normal_(getattr(self.base_model, self.base_model.last_layer_name).weight, 0, std)
constant_(getattr(self.base_model, self.base_model.last_layer_name).bias, 0)
else:
normal_(self.new_fc.weight, 0, std)
constant_(self.new_fc.bias, 0)
return feature_dim
def _prepare_base_model(self, base_model):
if 'resnet' in base_model:
if self.gsf:
import backbones.resnetGSFModels as resnet_models
self.base_model = getattr(resnet_models, base_model)(pretrained=True, num_segments=self.num_segments,
gsf_ch_ratio=self.gsf_ch_ratio)
else:
import torchvision.models.resnet as resnet_models
self.base_model = getattr(resnet_models, base_model)(pretrained=True)
# print(self.base_model)
self.base_model.last_layer_name = 'fc'
self.input_size = 224
self.input_mean = [0.485, 0.456, 0.406]
self.input_std = [0.229, 0.224, 0.225]
feature_dim = getattr(self.base_model, self.base_model.last_layer_name).in_features
elif base_model == 'bninception':
import backbones.pytorch_load as inception
if self.gsf:
self.base_model = inception.BNInception_gsf(num_segments=self.num_segments,
gsf_ch_ratio=self.gsf_ch_ratio)
else:
self.base_model = inception.BNInception()
self.base_model.last_layer_name = 'fc'
self.input_size = 224
self.input_mean = [104, 117, 128]
self.input_std = [1, 1, 1]
feature_dim = 1024
elif base_model == 'inceptionv3':
import backbones.pytorch_load as inception
if self.gsf:
self.base_model = inception.InceptionV3_gsf(num_segments=self.num_segments,
gsf_ch_ratio=self.gsf_ch_ratio)
else:
self.base_model = inception.InceptionV3()
self.base_model.last_layer_name = 'top_cls_fc'
self.input_size = 229
self.input_mean = [104, 117, 128]
self.input_std = [1, 1, 1]
feature_dim = 2048
else:
raise ValueError('Unknown base model: {}'.format(base_model))
return feature_dim
def get_optim_policies(self):
first_conv_weight = []
first_conv_bias = []
normal_weight = []
normal_bias = []
bn = []
linear_weight = []
linear_bias = []
conv_cnt = 0
bn_cnt = 0
for n, m in self.named_modules():
if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Conv1d) or isinstance(m, torch.nn.Conv3d):
ps = list(m.parameters())
conv_cnt += 1
if conv_cnt == 1:
first_conv_weight.append(ps[0])
if len(ps) == 2:
first_conv_bias.append(ps[1])
else:
normal_weight.append(ps[0])
if len(ps) == 2:
normal_bias.append(ps[1])
elif isinstance(m, torch.nn.Linear):
ps = list(m.parameters())
linear_weight.append(ps[0])
if len(ps) == 2:
linear_bias.append(ps[1])
elif isinstance(m, torch.nn.BatchNorm1d):
bn.extend(list(m.parameters()))
elif isinstance(m, torch.nn.BatchNorm2d) or isinstance(m, torch.nn.BatchNorm3d):
bn_cnt += 1
# later BN's are frozen
# if not self._enable_pbn or bn_cnt == 1:
bn.extend(list(m.parameters()))
elif len(m._modules) == 0:
if len(list(m.parameters())) > 0:
raise ValueError("New atomic module type: {}. Need to give it a learning policy".format(type(m)))
return [
{'params': first_conv_weight, 'lr_mult': 1, 'decay_mult': 1,
'name': "first_conv_weight"},
{'params': first_conv_bias, 'lr_mult': 2, 'decay_mult': 0,
'name': "first_conv_bias"},
{'params': normal_weight, 'lr_mult': 1, 'decay_mult': 1,
'name': "normal_weight"},
{'params': normal_bias, 'lr_mult': 2, 'decay_mult': 0,
'name': "normal_bias"},
{'params': linear_weight, 'lr_mult': 1, 'decay_mult': 1,
'name': "linear_weight"},
{'params': linear_bias, 'lr_mult': 2, 'decay_mult': 0,
'name': "linear_bias"},
{'params': bn, 'lr_mult': 1, 'decay_mult': 0,
'name': "BN scale/shift"},
]
def forward(self, input, with_amp=False, idx=0, target=0):
with amp.autocast(enabled=with_amp):
base_out = self.base_model(input.view((-1, 3) + input.size()[-2:]))
if self.dropout > 0:
base_out_logits = self.new_fc(base_out)
if not self.before_softmax:
base_out_logits = self.softmax(base_out_logits)
base_out_logits = base_out_logits.view((-1, self.num_segments) + base_out_logits.size()[1:])
output = self.consensus(base_out_logits)
return output
@property
def crop_size(self):
return self.input_size
@property
def scale_size(self):
return self.input_size * 256 // 224
def get_augmentation(self):
return torchvision.transforms.Compose([GroupMultiScaleCrop(self.input_size, [1, .875, .75, .66]),
GroupRandomHorizontalFlip(is_flow=False,
target_transform=self.target_transform)])