From 8f570cd5d0810d6e705ae9a553c24ee3d94bcff9 Mon Sep 17 00:00:00 2001 From: caoweihan Date: Mon, 28 Mar 2022 19:38:19 +0800 Subject: [PATCH 01/11] add rkd --- mmrazor/models/losses/__init__.py | 3 +- mmrazor/models/losses/rkd.py | 101 ++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 mmrazor/models/losses/rkd.py diff --git a/mmrazor/models/losses/__init__.py b/mmrazor/models/losses/__init__.py index c161c5684..8df360407 100644 --- a/mmrazor/models/losses/__init__.py +++ b/mmrazor/models/losses/__init__.py @@ -1,6 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. from .cwd import ChannelWiseDivergence from .kl_divergence import KLDivergence +from .rkd import RelationalKD from .weighted_soft_label_distillation import WSLD -__all__ = ['ChannelWiseDivergence', 'KLDivergence', 'WSLD'] +__all__ = ['ChannelWiseDivergence', 'KLDivergence', 'RelationalKD', 'WSLD'] diff --git a/mmrazor/models/losses/rkd.py b/mmrazor/models/losses/rkd.py new file mode 100644 index 000000000..db6873ee1 --- /dev/null +++ b/mmrazor/models/losses/rkd.py @@ -0,0 +1,101 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES + + +@LOSSES.register_module() +class RelationalKD(nn.Module): + """PyTorch version of `Relational Knowledge Distillation. + + `_. + Args: + loss_weight_d (float): Weight of distance-wise distillation loss. + Defaults to 25.0. + loss_weight_a (float): Weight of angle-wise distillation loss. + Defaults to 50.0. + l2_norm (bool): Whether to normalize the model predictions before + calculating the loss. Defaults to True. + """ + + def __init__(self, loss_weight_d=25.0, loss_weight_a=50.0, l2_norm=True): + super(RelationalKD, self).__init__() + self.loss_weight_d = loss_weight_d + self.loss_weight_a = loss_weight_a + self.l2_norm = l2_norm + + def euclidean_distance(self, pred, squared=False, eps=1e-12): + """Calculate the Euclidean distance between the two examples in the + output representation space. + + Args: + pred (torch.Tensor): The prediction of the teacher or student with + shape (N, C). + squared (bool): Whether to calculate the squared Euclidean + distance. Defaults to False. + eps (float): The minimum Euclidean distance between the two + examples. Defaults to 1e-12. + """ + pred_square = pred.pow(2).sum(dim=-1) + prod = torch.mm(pred, pred.t()) + distance = (pred_square.unsqueeze(1) + pred_square.unsqueeze(0) - + 2 * prod).clamp(min=eps) + + if not squared: + distance = distance.sqrt() + + distance = distance.clone() + distance[range(len(prod)), range(len(prod))] = 0 + return distance + + def distance_loss(self, preds_S, preds_T): + """Calculate distance-wise distillation loss.""" + d_T = self.euclidean_distance(preds_T, squared=False) + # mean_d_T is a normalization factor for distance + mean_d_T = d_T[d_T > 0].mean() + d_T = d_T / mean_d_T + + d_S = self.euclidean_distance(preds_S, squared=False) + mean_d_S = d_S[d_S > 0].mean() + d_S = d_S / mean_d_S + + return F.smooth_l1_loss(d_S, d_T) + + def angle(self, pred): + """Calculate the angle-wise relational potential which measures the + angle formed by the three examples in the output representation + space.""" + pred_vec = pred.unsqueeze(0) - pred.unsqueeze(1) # (n, n, c) + norm_pred_vec = F.normalize(pred_vec, p=2, dim=2) + angle = torch.bmm(norm_pred_vec, norm_pred_vec.transpose(1, + 2)).view(-1) + return angle + + def angle_loss(self, preds_S, preds_T): + """Calculate the angle-wise distillation loss.""" + angle_T = self.angle(preds_T) + angle_S = self.angle(preds_S) + return F.smooth_l1_loss(angle_S, angle_T) + + def forward(self, preds_S, preds_T): + """Forward computation. + + Args: + preds_S (torch.Tensor): The student model prediction with + shape (N, C, H, W) or shape (N, C). + preds_T (torch.Tensor): The teacher model prediction with + shape (N, C, H, W) or shape (N, C). + Return: + torch.Tensor: The calculated loss value. + """ + preds_S = preds_S.view(preds_S.shape[0], -1) + preds_T = preds_T.view(preds_T.shape[0], -1) + if self.l2_norm: + preds_S = F.normalize(preds_S, p=2, dim=-1) + preds_T = F.normalize(preds_T, p=2, dim=-1) + loss_d = self.distance_loss(preds_S, preds_T) + loss_a = self.angle_loss(preds_S, preds_T) + loss = self.loss_weight_d * loss_d + self.loss_weight_a * loss_a + return loss From bf61b6dca3d4851512a82441b37b9ed86aa6c131 Mon Sep 17 00:00:00 2001 From: caoweihan Date: Mon, 28 Mar 2022 19:38:41 +0800 Subject: [PATCH 02/11] add rkd pytest --- .../test_algorithms/test_algorithm.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/tests/test_models/test_algorithms/test_algorithm.py b/tests/test_models/test_algorithms/test_algorithm.py index 4ec729387..a42886e15 100644 --- a/tests/test_models/test_algorithms/test_algorithm.py +++ b/tests/test_models/test_algorithms/test_algorithm.py @@ -436,3 +436,105 @@ def test_cwd(): # test algorithm train_step losses = algorithm.train_step(mm_inputs, None) assert losses['loss'].item() > 0 + + +def test_rkd(): + student = dict( + type='mmcls.ImageClassifier', + backbone=dict( + type='ResNet', + depth=18, + num_stages=4, + out_indices=(3, ), + style='pytorch'), + neck=dict(type='GlobalAveragePooling'), + head=dict( + type='LinearClsHead', + num_classes=1000, + in_channels=512, + loss=dict(type='CrossEntropyLoss', loss_weight=1.0), + topk=(1, 5), + )) + + teacher = dict( + type='mmcls.ImageClassifier', + backbone=dict( + type='ResNet', + depth=34, + num_stages=4, + out_indices=(3, ), + style='pytorch'), + neck=dict(type='GlobalAveragePooling'), + head=dict( + type='LinearClsHead', + num_classes=1000, + in_channels=512, + loss=dict(type='CrossEntropyLoss', loss_weight=1.0), + topk=(1, 5), + )) + + # test RelationalKD w/ l2 norm + algorithm_cfg = ConfigDict( + type='GeneralDistill', + architecture=dict( + type='MMClsArchitecture', + model=student, + ), + with_student_loss=True, + with_teacher_loss=False, + distiller=dict( + type='SingleTeacherDistiller', + teacher=teacher, + teacher_trainable=False, + teacher_norm_eval=True, + components=[ + dict( + student_module='neck.gap', + teacher_module='neck.gap', + losses=[ + dict( + type='RelationalKD', + name='loss_rkd', + loss_weight_d=25.0, + loss_weight_a=50.0, + l2_norm=True) + ]) + ]), + ) + + imgs = torch.randn(16, 3, 32, 32) + label = torch.randint(0, 10, (16, )) + + algorithm = ALGORITHMS.build(algorithm_cfg) + + optimizer = torch.optim.SGD(algorithm.parameters(), lr=0.01) + outputs = algorithm.train_step({'img': imgs, 'gt_label': label}, optimizer) + assert outputs['loss'].item() > 0 + assert outputs['num_samples'] == 16 + + # test forward + losses = algorithm(imgs, return_loss=True, gt_label=label) + assert losses['loss'].item() > 0 + + # test RelationalKD w/o l2 norm + algorithm_cfg.distiller.components = [ + dict( + student_module='neck.gap', + teacher_module='neck.gap', + losses=[ + dict( + type='RelationalKD', + name='loss_rkd', + loss_weight_d=25.0, + loss_weight_a=50.0, + l2_norm=False) + ]) + ] + + optimizer = torch.optim.SGD(algorithm.parameters(), lr=0.01) + outputs = algorithm.train_step({'img': imgs, 'gt_label': label}, optimizer) + assert outputs['loss'].item() > 0 + assert outputs['num_samples'] == 16 + + losses = algorithm(imgs, return_loss=True, gt_label=label) + assert losses['loss'].item() > 0 From d000fe021c6ec5d8edcd4fc6dfed8656323b300c Mon Sep 17 00:00:00 2001 From: caoweihan Date: Mon, 28 Mar 2022 20:19:06 +0800 Subject: [PATCH 03/11] add rkd configs --- configs/distill/rkd/README.md | 42 ++++++++++ .../rkd_neck_resnet34_resnet18_8xb32_in1k.py | 72 ++++++++++++++++++ ...kd_neck_resnet50_vgg11bn_8xb16_cifar100.py | 69 +++++++++++++++++ docs/en/imgs/model_zoo/rkd/pipeline.png | Bin 0 -> 48004 bytes 4 files changed, 183 insertions(+) create mode 100644 configs/distill/rkd/README.md create mode 100644 configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py create mode 100644 configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py create mode 100644 docs/en/imgs/model_zoo/rkd/pipeline.png diff --git a/configs/distill/rkd/README.md b/configs/distill/rkd/README.md new file mode 100644 index 000000000..84de61ef6 --- /dev/null +++ b/configs/distill/rkd/README.md @@ -0,0 +1,42 @@ +# WSLD + + + +> [Relational Knowledge Distillation](https://arxiv.org/abs/1904.05068) + + +## Abstract +Knowledge distillation aims at transferring knowledge acquired +in one model (a teacher) to another model (a student) that is +typically smaller. Previous approaches can be expressed as +a form of training the student to mimic output activations of +individual data examples represented by the teacher. We introduce +a novel approach, dubbed relational knowledge distillation (RKD), +that transfers mutual relations of data examples instead. +For concrete realizations of RKD, we propose distance-wise and +angle-wise distillation losses that penalize structural differences +in relations. Experiments conducted on different tasks show that the +proposed method improves educated student models with a significant margin. +In particular for metric learning, it allows students to outperform their +teachers' performance, achieving the state of the arts on standard benchmark datasets. + +![pipeline](/docs/en/imgs/model_zoo/rkd/pipeline.png) + +## Results and models +### Classification +|Location|Dataset|Teacher|Student|Acc|Acc(T)|Acc(S)|Config | Download | +:--------:|:---------:|:---------:|:---------:|:---------:|:---------:|:---------:|:------:|:---------| +| neck |ImageNet|[resnet34](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet34_8xb32_in1k.py)|[resnet18](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet18_8xb32_in1k.py)| 70.23 | 73.62 | 69.90 |[config](./rkd_neck_resnet34_resnet18_8xb32_in1k.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_b16x8_cifar10_20210528-a8aa36a6.pth) |[model]() | [log]()| +| neck |Cifar100|[resnet50](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet50_8xb16_cifar100.py)|[vgg11bn]()| 72.76 | 79.90 | 71.26 |[config](./rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_b16x8_cifar100_20210528-67b58a1b.pth) |[model]() | [log]()| + + + +## Citation +```latex +@inproceedings{zhou2021wsl, + title={Rethinking soft labels for knowledge distillation: a bias-variance tradeoff perspective}, + author={Helong, Zhou and Liangchen, Song and Jiajie, Chen and Ye, Zhou and Guoli, Wang and Junsong, Yuan and Qian Zhang}, + booktitle = {International Conference on Learning Representations (ICLR)}, + year={2021} +} +``` diff --git a/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py b/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py new file mode 100644 index 000000000..0fddb6134 --- /dev/null +++ b/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py @@ -0,0 +1,72 @@ +_base_ = [ + '../../_base_/datasets/mmcls/imagenet_bs32.py', + '../../_base_/schedules/mmcls/imagenet_bs256.py', + '../../_base_/mmcls_runtime.py' +] + +# model settings +student = dict( + type='mmcls.ImageClassifier', + backbone=dict( + type='ResNet', + depth=18, + num_stages=4, + out_indices=(3, ), + style='pytorch'), + neck=dict(type='GlobalAveragePooling'), + head=dict( + type='LinearClsHead', + num_classes=1000, + in_channels=512, + loss=dict(type='CrossEntropyLoss', loss_weight=1.0), + topk=(1, 5), + )) + +# teacher settings +teacher = dict( + type='mmcls.ImageClassifier', + backbone=dict( + type='ResNet', + depth=34, + num_stages=4, + out_indices=(3, ), + style='pytorch'), + neck=dict(type='GlobalAveragePooling'), + head=dict( + type='LinearClsHead', + num_classes=1000, + in_channels=512, + loss=dict(type='CrossEntropyLoss', loss_weight=1.0), + topk=(1, 5), + )) + +# algorithm setting +algorithm = dict( + type='GeneralDistill', + architecture=dict( + type='MMClsArchitecture', + model=student, + ), + with_student_loss=True, + with_teacher_loss=False, + distiller=dict( + type='SingleTeacherDistiller', + teacher=teacher, + teacher_trainable=False, + teacher_norm_eval=True, + components=[ + dict( + student_module='neck.gap', + teacher_module='neck.gap', + losses=[ + dict( + type='RelationalKD', + name='loss_rkd', + loss_weight_d=25.0, + loss_weight_a=50.0, + l2_norm=True) + ]) + ]), +) + +find_unused_parameters = True diff --git a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py new file mode 100644 index 000000000..6b6d2a64b --- /dev/null +++ b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py @@ -0,0 +1,69 @@ +_base_ = [ + '../../_base_/datasets/mmcls/cifar100_bs16.py', + '../../_base_/schedules/mmcls/cifar10_bs128.py', + '../../_base_/mmcls_runtime.py' +] + +# model settings +student = dict( + type='mmcls.ImageClassifier', + backbone=dict(type='VGG', depth=11, norm_cfg=dict(type='BN')), + neck=dict(type='GlobalAveragePooling'), + head=dict( + type='LinearClsHead', + num_classes=100, + in_channels=512, + loss=dict(type='CrossEntropyLoss', loss_weight=1.0), + topk=(1, 5), + )) + +# teacher settings +teacher = dict( + type='mmcls.ImageClassifier', + backbone=dict( + type='ResNet_CIFAR', + depth=50, + num_stages=4, + out_indices=(3, ), + style='pytorch'), + neck=dict(type='GlobalAveragePooling'), + head=dict( + type='LinearClsHead', + num_classes=100, + in_channels=2048, + loss=dict(type='CrossEntropyLoss', loss_weight=1.0), + )) + +# algorithm setting +algorithm = dict( + type='GeneralDistill', + architecture=dict( + type='MMClsArchitecture', + model=student, + ), + with_student_loss=True, + with_teacher_loss=False, + distiller=dict( + type='SingleTeacherDistiller', + teacher=teacher, + teacher_trainable=False, + teacher_norm_eval=True, + components=[ + dict( + student_module='neck.gap', + teacher_module='neck.gap', + losses=[ + dict( + type='RelationalKD', + name='loss_rkd', + loss_weight_d=25.0, + loss_weight_a=50.0, + l2_norm=True) + ]) + ]), +) + +find_unused_parameters = True + +optimizer = dict(type='SGD', lr=0.1, momentum=0.9, weight_decay=0.0005) +lr_config = dict(policy='step', step=[60, 120, 160], gamma=0.2) diff --git a/docs/en/imgs/model_zoo/rkd/pipeline.png b/docs/en/imgs/model_zoo/rkd/pipeline.png new file mode 100644 index 0000000000000000000000000000000000000000..6763cebd1a5bb3aebe5868547654cc97e8d58ef4 GIT binary patch literal 48004 zcmdSBi$Bx<`#-M3D;<<{lH`zb$gv1HF2@`~PKmIRV~$M{+E%F)Ip>rRi6WNMn8TKk zQwU|)SY?i5a@sK4eveVF_xtnte*S@<+pSy4_SoaPuKRUe_v`TV^rHFs-8+x$Y-z%!^Kq+yKC**Kj8Zv z0Y>(LTwHq^IDa-Z`@ePN;;Ioa*4MGQ<2=KREjwWsk{U~bT$MR!YxKr1>0Q;0*SgTl zy4Q5}9dLoF{Y1PvR6fsZuT#?8USYjUR$nyjboz%6TOSr24)ne6axJ?x*z$f{UFQ5Y z1D(M;Iiibv&VjzI;x?-IWBN}z_8!|+5>($FieVsDGbWXKdjqTUkzJFSA_R4GqH~srnu^Ger_upJx7nR}||M`wfdCUKAeyE&}r?j@U?bO*D)704+ zgG|o5?w>D186LMIz}04PhuVkFq0LEhGZo&sL&#wypTuI%J`Z!O2|7P@Y{-H3}Yoz;@VLMv+W*r-4Zg$4?02=o-`LDE1%ZA58sv7}(($z(0q0kZZV>CScN zP`6YW$YYRLg&Et~g+TYWUCE zf8PCSOq9Nf=z=9e6g29tm#W+)RUf5f?N>UFbbnkO&$JdBl&!p;GUP7k4tZdy(Cr`7 zds}uzA2}6qNMrl|I^@%K&l)10drUZG>ADAbpiF20H7nHBgF6vE9Pf40H@VN=KC!cp zy69bEN>8m|M7@~7o|E@A_Q2Vh2c}4y++XQ-t&`DEb_x~w7o`^ifwKJg6zGzUgGXAK zt)!M4-3@31K@wC9XDvc!Q(T;n*^w2%X7F`$VB>uQX_s__3qN=XuhucWs z(-6M{azEoa-gi;*bEQzszK(0(5Qo+w$Gd`jOUlz!(!(-r6=B2^l{ za(K|8Jzh@7i-yZs-g~SD=XJrSaorQoeox>kbh}F#8_e^WnXorCge~gUi$3>wb#!xcDI8z~R2M0bM6Rz6t zD!F~ZuU&TWm_A3c4*?u~#~V@8mFJo%ln8tuDRF2fvB#l8Hvh&$`nSQgO<$Zg)%*Gm z9V@!-;rbz8i6)Z3icyt5>euXxfrO=eiYooCCf&R`e(Sjm2TyrOys^)CSL;AWUjVVD z6n0VliozbqwMX1jAZW?%;)uq@6~Ir{zIWwB0z=Oj{xyuW$;($;B0LHkhZmR{56&g| zwmW=~1@RXT*1u7bu5Wn`A)i?fo3Zo)`w_x56rb{1E%y&FbezNa8t8U69ybFpr< zGZmul+v?yuvL`jZLXWoIW&17GWh9o~oPMSmb_vPVKU;_BHCkmbgc-H>(?sm1xYrH7 zXGGscVsj4ZggIj0?Yxn$1BXfKm~nCISR?<`muA|yK5C4+>DC>7l%8|wP$4g`Pp65_ zrs^KIW!ooGEB5hTt`vgv)L%IhB*<$NeAhdVe-&OQCvai(=2*lY=5%0bpnER~x23c@ zEH4jomm~Q%L5}39r)_JAcAf3!Y}UuiosFBkf~Z+S3NdMwd017fM%efrSpk6v&$42+2A)5cA&EOi-^9f=76%Lo zUBkzigh`(i6nE$*{1z*}?Xrlz@KC&Z*cl_R+%3A7_2S{eC>9s}th};3Ylnh@e(07lSQ^=;|)SJ%5H!ttQK$jip*M4R3o)SSm zQ(w6Ht7<0grf=Wutt}jUof8|AZNL7oRzkZ_Za`DxFxJEW_wJR?RPouaX16+c<>Kmn zd-})#ced}g%8F9?nM4nG3Bnm&BX~SbJLn2oA069w)aKS9R$b5VSplO@IVqz8l8>?n zj)sLdy9O?A(F-WjU4U03Su?L!g~Ga8J5?{PC%Y&vPE>t8XLSVQ?_QrD-@z)?{<-72 z*OTWA_s#t;gDAVM&N^DCT(Wa33haZWgl=9N-*w?LhLx)9j>+f_4$T^vXvNH>Vi3Sj zxw!Vl+fcdxVch+!G!?FN>=#wYA_V<|^A>-!G^PN9O-NMCnLhKzeHq)&{wLhUZe=}vAyFr>RM<{J$ z{wUR?^iXMHeQwX<7$d&hXw8R4;4fR?-u4{hog3WZ!o}ClIVqfc zX*@mO3MYJlucx4Uf`86WP-ss~th?`%fxR;5jn~(_%Kunt&F)Iiq_6VWo?9Dum~Vj} zZ}7;P62Zw@9Kk?zX7^46-I7E43tCs+>P%#G6JBmxkNrQ6a6;N!M{0xB15S)H(%*v- zIo{PC$2k7%9!~v~_wgGYa6;I~x?;GhK(ziS7st5x&w8lT-`ZFeaC#(uYAlxZ{R%?nrzPnsw4(X5J#G z{)67&4*|P59x6)D-npK(jRj2U#?QnBop98qwb=Q^O~CmZ{{jh9!Df zX$A9#(HHJf1hGe>4>yU>5*?yd<9wroyn-3l6h?D9+<-pEdKWPtgAOz?_usvFA4%5o z`7MFuIGJo|1S%=XLaZ2BmU1hQBtbfOYbap8|1LO85)*% z*Os8{3R2G|iP~BCtY8#;f5aZ4dG^kWvKz4(AV})yK?Vvw#E^^?c!PYw8I%C1xofia*VGbszQG~m0UZ-drF7!gpcuMFhUVf z70&AbEPsC3rg=u7jA!4BpwCv1Staj3%31&EqwDU)ie&;XwnLojHOqNj`?R0${x_fR z1dJ@Be&^RS&xm+|wXZ(6#_DJuUL$xU6F+K)T(T=dMn%!F^g5R6mM?%{tslt6_2_lN zP7JSElWa}S%+VK7k#IQn!SMauAZK^Ywcia5DsMK$9dOw7=fwy1z(ig=Lwl1EWv5-( z1QH9)HVK&st{qcG#nW+YW((ZtpCV3y!1>yFJ|*`YRnJ<7s+w-_lnG&!LG-A4(w=e~ zKL$#Xej8`=)h3S7;TG=#$c=W{*74H6&0Xkq#TasjeAT)smVwFuTT0>vV9+#rf6m6kv2l-rLFu%{|GOnZycDF#A~SP)8j z&H*djJN=^k#)D9TZ)L~!WkucnGW${{@2BtC)Amd<^Nx!nb(M5+PoB95V)n-c{0cr@ianyDmqk zpm3S@jf_*(i$XuN{@BOGB`Q?>g!$o1p(4a4rxE)&3Pw6ug3@8x@=l$gDVM|f1wvC4 z&H%;)Y<*#x!lj!S?FP=ea&g)lQ|TbLO-%gi)GfYnvu!EBpshEJr@d2OR<*5Ne7c`; zq_`EH0z_O%VSEEOJbV&r1E9xotgEf|x&5k7+ZC-wKg!9{Dv0rnrt4F;EdL%80AB8C z?&yYi`vn{$-wMaCA4B+N3643IrFu)l{mNHIP8cx#lGB7Wc1(_z=Jr!5LER@R9J0=j zI_jfXwUvu2H8#oxsu$=2-u+kxMl#^YH7Xx;ka5HyIzC7P7}1>*buU{v?qwKA$OqLM z{V?Dl1#vS~-D&Y^V!b#B9GbdHk+jJ&QEJMiU*B;b=Vry%Pkh+LL1mVRb$&6Aihv_PRlm-3 ztdR^&`Orza6aV^sy0?~~EWaJLuY7%<=&#bRaPOgKglA&Q!*t7UO&`3fBxuUX*~QPc z-ZtRCZ{gb1w>R|?;ig1Z!+;nV<7&uRyP7$Rkx8R7`CtaYg-y(>P`#~&u zT&R6cw_LB>*!5Ef=V~V$fg6~iWRBl=+Q*S}I#oT$cO6n~4>YTsb%ZrHl|aM_`|+0_ z5>Zk8cu_ViT;eG_I%oCfP|P+MZXn=0x6zZ^PtfGF0Hw(4!z%ySHDL`CDQ!kqIo5_j z1oOaE{;2vbHU>@<$`2J5co(kG%DX~{eq4g0y{2k+Lt4gBNrk;e3m2s z*zBWQ399>Bwc@^hVEOw*i`aM8cHHSa@|#}CwuDm-lIx$q^_+pm-VjmgoclS*ngNjv zmPV?D00zo_7VyUHo!RT#s`S(SK+x&;8pToYXob%11fhN*qgu1yHDCvqn~ty} z>M~99a0w)5J2)_jhpaNy;DG74isPz zH6XAE2e}&p%LWQR>ld+n-j9E)OU2txd&eQ=9!r2b{HwvMIIjN|IOvT$P=|4l+ldd9 zb+GVQ=La435!SwYSr`P4Yjy6?(%XzK$~t(Ut7VUuM;m4%%iwv>US(Y34GE}ov0u2_ z$a*3Mj4%Mhb#e}XYMB9G{d<2eqAk=yQ>)c%i@b(UMirktnx4W30$m}y!|zql-lO-D z1VH+Dz%Ckn?dEHmxkFWt;%$E#e$i>$-g=g-cU9c`REh{H~*t44uG3KmtAuIP9<8FL5W6qz|ft$k(Xf1N9BwclD z7GeDqG3+}p;E5s3%YEwjy6T{E=olkeXWIMuNqWU_d0n44zVwkZqhLBVCBxt)#rCh> z9*UbpRJL<+>Klz=Z4<2gVRH}OS>sJve`{}2!;RRuKQqy($&Y$-$cVF9xc7!kwY^k|GP(3+DL)dX8sun$ z8MhBac8@ws*p41B=??GE3~!gfd-ZY*29rNt;+Xo)1qQ{FvDB>ON6E%%o)mO(nGeTm=*QrS$_Y~M|RYCI{GR1&=Bu` zUs|vPHF}SzpwSZUln-=Vvh>ul`wX~|EAsVH-b`6Bz}Gw(dMXR}3dmeDx5Mt7_v1_T zmH`NRxAX^Lmi+$+k=L4_YfYcFsU87+0ia>u3|Sa%`szjN1EI&hkLy3bnQ3$KGPn`Y zvk9coGN{z07-IXp=G{n6yATFqW~=tds_&{&*N@Vt03PMOmrhqHtJz(4!JK+$&>)UM zG4kEA-K-=W=sMtT8W_laa=qK*KD)$s1%RHc>R&kn7{Gz=B+oi8!u!BtwHK= zfkOvNpf-LwF}S1OXay)q=Y!$DtD-*PR_)j~4o4j!ddgZ+?SgK`Ou2%#S5|yS7&WOu zmHDMg&v``yC#8CK@z}r`Y`Jcn96U>D?WT()rKHhA3Ge~4tTCDGtd#HQ?at`N^X#$EX$h=Bj1ek#nilZ4XZ3r4#?!H^maz zcLxWQY3rcO`|j2I6>ZS~$~W=oU}s<4;u9;iGVYmt$D*e6%AR-Kp^{ z^IcVE(aiG6oj(~8;V)9^=EmR8W+nahqA^|)Lz-*q=%Fam9sS}R!v)OXVL9Af+)P9` zIh$Zv9zFHQCqA3`J@@FP$82WfehVJDg3KVqRvdtddv9KG>D4xycFfMdUV901oPORW z$y&;dS7m`mKHVnYOplO8&!|h!qaD$hTi8`@_3kOHlEZxbDzi z70X*;A1rSdyX_()=0NcB&*`@_#2m`uK zv?fjy62_CMxd z!@JN3EZ~R7Eu5kUJ(OD`|Kz=P*v%0Q*dq zA;pZK!{>Wd*p}>oK1bzm2kA2}65h1f#EHU}U>m9PCjUQmPpUz_0n`%2w8UGbwykh8 z*Ycsso9p^letQ~45Rp(hG%+GT>pq>?jZ%8>I6U_0#Vq#9$5#@!^6$Y(z0}}y)Cg{k ze;)Rw$PjN6d>d}tG&VU@cAo%P$S+Yk&fUe2rQQDK5%?XbuyZHG+W1hA7?hB=o)%-a*EClXyibJGaXZ*{D%US8qH*#1n z2fd~Jb{8P;nM!9;2EpYx39Q-afTJMM6PUBG`mE%PYkM-I+9$H}%;}8m$8sT^!hGQ` z`UlS6Pd)N>;@pp8A^2}IO=$XOo?LgaI~pM5CeP#!&)y^Q3Yqd{lfIuRNA{3QytX{i z400Q^P5T#Ag;I1kU>Oh!lh#9F_22wlTn(s@K`WpYTFq@o?h2Mu$3clvI*O$>>&{$vk+Z71FKC|Dg;U3S&0(z{NY^oJCD`v6NEY)#44{t(LWLB44Z8&E z7LN1UE=9D}YM(yb4iHhg$jRi}7D=V=6!E41z;K5T-7HphnwH6fNHqJaRD6Jj{<1y2 zJ+?-w>TFf_TN94=n)kLQpQI~?oX)s<%BMAu)*?v{SDL-yKMpCsx6h0YL$;Egx|R zA2O-L!W&xxn&@VZj z!~749`b|}}!Jvj1sI7UntjmX|L=46`f z1j6kbv4DO6Pyakwh|RoX%kybOey2RI-ekNeLeR<@2=NvB&v0tTPGQUN zBTA`8%cqJCztfhXw;%nMOY|or#LAI3O8fCG)^=k=S#|r<_$=wRp>H?8k79j5ntZ8i z|Ly924Ag&}@D|2^I?M!+neiMzRj*LJh2J+9!ynjZHYfYL#lF=%JGJz7I~nb-d8~4{ zQfVb!{JcDJ4AH}+ZN`Wg|CLNJlF42cVUK*!noOuz_!`xXQM0{8J zjKhT`N>CGO?{s2FJ}>L0{wDDst3M21U%)`WH>r6`UdbccspT~Lop`$94+;RSoNb%d zbM-NWUh}hS=ZaC&EzQ)b^Lt2(f&1@k3s*Uc<7?Ttm07r<%Ir#R?d12>P){x;VWT^< zNILSx`(vSFr&po@MY4PL205r>*2XlXb{R18^FV>68Kmq(az2pu*V=dZMzI$*v_s}- z2fCr1VxS#qj&=%wcC4v|jlHao<;?{PKgjzn&$#4P`)+>L8gBg_lXZsIyprTBN-z2s zvt|7j36ROK1gxq;)R&3}*BTKAeJh@F6akt)6CoRlsQp(F7bEJU4ei%yOr|+n7K576 z^^lr;_D0k}7)@3SYveJ+&Tj{La09U!R1xJxIsbBT1>mp-e*^l#`$UxF2CzYc2sjP^wmks0IDIL# zLhEs55i?wl)Fym78{-K`lTBY1I)C1YUWEFND{Uzd}ir;G5PfG7f zeyjfyQzVi1CE_#ENiyfI?(kylFvXP36B1Cs!3{+34{o-(K!o8IJ@hfRZcOryMuaxU zsb6zQhYlbeBb*YRQ!L)(e^ku_{9wIGTDV>6)(S5>YEjkuU9m*GYv8y!)q6s8r1)!i zJ;HE!kySGah>$=i4kCJ5X(=K+GK{=D4pBXF=1WUp#sIQw5gmkPp4XOaZVrE0#f)NH znP;k08cJsVd@i;aMu|k%x$$jitmHq{i$W@xc*Qs(;3{ZWmteYiZIv=p67cx-D#?oiYHF4vy)Oe&j9LGN*z_gdZxDNS~ zu~vV`*!}+4^Q6SZx{#a4@IB|I#~aGiz_8*@YGtxOsN|qx6x{6=^H{ULy1CO*MWFt4 zepJpV!j|77@#5&DE0_RRZ&N9_TC=fQ|L$4EZG`n6Yk&6lRD z!xm`~67422q%e~`u^v;D_iH1t#Y{-@m52w3OPQp=)njVv6}}mV7eqGiw=6Y>8JCdU z2VP-Jx4z+J4y#-KKeCGBb*IPA2b$IOP2t>{uME#03c=S5yA`z;xWv0a1)L&U=F(%7 z?BFv;`52S0jX1$#;Dk*GQcKQ(>P`46z_8pRE+@F&t8#z_=@CG|23NY*5(#9s_$%{u z{x&D>LVFRGtv)8j7>Eu?pN)VSsy8p&cYjHiTw4O88W{v^54xRNe!hw1`i|{IsOpRV z=VkfB$6nY=zRop($2%+dF4-H%I!w!iX5IirzMy3B>Y&=C^(_1CCdft4*P9#r3dH=P zqyeFXX{x`C(rs@}*4*z0ip7ktEHC>e*6Ud9saf05#f2mG|JGDBVzdRG3-ifgX`I>t zbW+ouIfaA!K{3-x5~mz~44?-_p?E0=>Up{5>t8vz@_RFDI6X}s zpt@Jg{Vou}>fO<@|8k>rQCgDbM)x98CcGmLP@}Ktr~O_0KD2F`!bWT4%WD+Pk? zaX&TT@Z`ilLa4u*;VsD$8j3L-bZv4(7DxqKH=FS>O!{C7iJ)I5E{}Sv_+s5G5-t$r z0Z=EfrvOC))9=vfLHXA3^n(sidFwwIvb+M=wIvm(x}+H^3pRgRNpdhK#dp!UQQI|t z5=Ig)L8S)10DMMvx&W1NeZ4WDJH)MeB;>EEPp+T6kUVnQ(|#bqgVRie3C(2fWvqPl z^~yE=+bALe9`AaRuJ}$m|DvOSk4l|+`eSs5w0^Q5t`+zEGx~!jeEG?&h3z_?ZL9ktn9efg^d=M^oxvIgoP`? ze0+n-qraAFu4euSYi~^AD0)w5hgShH*lb#+VPRkwY|Xtat`K`W<_Xh9tniO7fH{aE z9}Ln8EDg`vP_;rzH)}^pmdH@1Vb6b5y`Nz+xS{H%q2N(v&t4i8`%8)|_!$D9$fL&dlIv-vY5jh}W=RY%o-^ojigs(8 zqi^#VbqyPW(6;nUX^YpeNH0uAIgzGtyJk#wi!l9~STc^Vw0o1&vFO#_F_x z+3?_$LQjI;G=IaygACb!nb!U$Y(m~fqw#6GOu@4apxZxc>6rnf72hJm!HGPG%WZHi zgl{@8WbFgnMDssZi2h>*o<;=FZ0>AFlnmyO3(oO{6la_hnTb1Mnm97+YZ^R++tS(? zYImEqhEsiR!ol=LLN@H+kyhu^F0u(b!3FN$bUeKhu3L7?ajKKUHrnYgnToVrZ^ z1&mDz>V(|?1sEjYrG~-)FnBPq0pd$K#WcTShMS3T!dHrE>gl(v)Lbjnm#-RDJ(2Jk zejkFgx?@{=BH1Eu_!iE$+4`0!EoDm`%@b~?a}Y)XV~z>6_ovw}aFXMUX4fD$+rApY zF!Fn1T}JVH%4w8uTVJtolYzjrX2Ta|&f6P%g;lxy2Y(nIk0z?}kC#$D@+h{0!AA%g z>wKYHw#rj4>G0$g1CiXSGxwzQre%!(WjU($(=vT9l7=Mn`WUS_yk{V^n;l+n`hHc? z)pwb0Yp-$Ja9j5K(|$#GQBJMq=xpgGn`?edX;6iaF{*!wBk%zr@Of&!{r?bnqQamH z!mOOWvye`b6PxZ@g{tqeW2h<6t{`k2Mi zdl}zfEe##O%vSf?C5$igyXJ?5(}&gD;R{=uBHyDKuwi77z*}+w^t-60AJ=drV1rf| zlv~!AZstLy7TGJH$W3Aln)k>yZ@IX2ka|X~IEUA&U^*fv$EO7KP&W=XP^0HKUqPGWn)W2FRYG-dyR9>B8Kp^>=3`^hg>T}HpKQ}omNeTQ2;*paXy#inVEi|2 zp}nC})}K0eH6ntl21RnC)t|td9QTinFSjt&tygVv1v+<#T($4Q1*1>EpzYc-k2H6= z$JsyX$A2|fk0M&s9OXUdT#nS8PSJDxo3}>*gN>{p$^i2B*G%Q+$CTI2Waz*#6G(;e zwkCN_G2E}Vmv-E|#zS=frIhoO0Q$%+S9_O13BCs+BUw5G@YI;aJyl6kr^1EFN@$r2H?6mL6w>qaU zCwF6{!sSp|o*$?vnoK(b_3M1IU+mH z+qytc^N$AnM;{=ERS@Z!@xza^e2V;xf;BYl#L8tY#-}gz9IBMH_)Beep{$J_q0Jf* zEAfV)kEINn(XR0c_Q@lgq&C~RIVUej$wVbE?3=JwZgp}d)Z?_m=I<;~uW!b{i(qT| zF!o*OpmY^yK~A<`*>HbpPUX+0Wp1vl7T>h(ri)qwLxVhPo_n{muVEHOSq5r&4fRzW z7l}`%pS2a3wC}?d3e1#|_pYld^!zz5P}QtkeiB&zEJyBW)Vkc#P3v-#vFL-zBlo4d zRU-No_ygm2OMvFWYd6%>>jNb|F3^&su(VY5?<#(U;W^g_8nISE87jLwDu6@j0}o-~ zLF|gZWO75zQ_zfqAigT;g`UjwAzw#O4%Bgwtdo||+cB~IP@ZBMw{ec23)DDsq{}|J zd3(sv@C{Ryu04-u2L$ixVN1eV84EY<{Yi3lh^u|f;d7mSQ#|aiD&YhN`xIJh8TK+n zfiSe29dqx8)A0TQ#Cv2XDW>xi&TddpG6IIwUo6!$WTC*)^o1Yh59;c#m#VGZKeqhk zEN2mqJGWw`-;+(tOw91FdP_4+V7TvfM%(o5zmW!VA;`m}_{W!T3#IC~e+O#-8k1*W zP=CF?hdW7_M*c{kZnI0mCqcb7bY=6*3E7!N9SWiT`zE& zp&B7;pS6SS%B=aO-zy)jqz3T!6s)Lt%W0X`z_nZ$M5=#WDbsSc>x0O?O1U!Eu5U1P zLp1k4wb4=-5l8oCpTJL-*J2^pu?m{6NlrUc7#wm{xGNpV)fINk;(>sxye?Rikj${5 zQhZw+BphqAO%^v7D3Q;EoYX*2DiMV}56>Qr0rwyZ1NlP^ao=OPnb8ZXA+^?z=jUfrpH^TeePq-hQeN*|-Y3U&4&ZYqX_9WLwoX<`C~1)#16D3DBI+}0_c-i;(JMo|vzVPT<;A{#en`O{| z*GPB0k>*bMc{xwrck>JHW3{A>!bvV#a!-;jbS=wEv?8~Eib4hFZ`b1*mt<`5tr8I!d+dJPB? zwRV-YnwA;UJ$hjseCK@mV959l%h-g-+bUP_G?#w&8tQiR7HDGSNLG02w=&&0y08$( z^pzattvZ*4p95V{u(9&SYlYv1wEO)`Tg$L_-bk7Y5a6cH7U82Ymss>Q<^SuPgir zfDAP2Jteg)s2hsoRy1e=RMW+D8E)-~{6dZPZd|iN2-sxFf@yvzv z$XM=(3I+$l4N!gvUOl4-<(o8m=W2J=i~W*nv5~s5IMAgk@3(vL9C?g3bPerf_Heol zES&$-7~nkrES^)^00I7M7n4_>aG+ugJ5zU4Hk*j=u|17y83`jXrZGKxpB zIvCuu^@~<{XY7Rg76MKn|KoJN?@xUGC%nlTNhY*M?u zJOsn{(V2E*i9yeZfZd$vn%m-18)SuEI-guO?m8}r6|*=&4(B!KbGt=NqP@UU;qtLW zJRMc}hCrIDoGD>EzZGG+cp{#0Uzq{cSJn3S$Iy#25Ej)l>6;1miU3i^U63W&Lbd|@ z^QYY7CL=*+zaE>&0QZaf{Qdrg;Q)ha8MLbbH@EGcZv_dAVxy}~m#A)5qZpg9ABsoY z9rJ^>mOyG0qMp1WLNBd5H?hLJ%dOKR;86&EuKP>pG;arw)mtK6ytn*B+hbv*01PKv zf@y*UkYq;c_>odB@5<8(zfi z!}#~DFA(x5BJWJf)9=xlF9iv7P#D#6o-*n4eI{0=V0wbiX|gD&-1?dy9U!po6PB>7 z(K{di8Qbj|nJbN%O#mI}rn9NaD)t zpEZ7uI{U<-E4Gk4y$5i{}d#op}2h5pbd0 z_Bp+>89UY7ni(0`@u8xn-2b^j`xX%ngHXAJD5P6B;W)prArksC2zJbOOUw%Y$Ye$& zb%bfnJ1XQm{`}?ek*%wWfvU!*A?kKI5`WBFGR*<67l`e*U(X~G;;wZ!^14R8*@>Bb z0kmPDwyurPSC?cRaBl5z{C~7@JwRa3RJpS#&4cb1pABR4BYyHHb37K}RQ z@bk?h*d{E|I(()VgCkGO@V=0y9SlSy2%br|0b%XD(2ofNV-1qxaj@e#aZiulK*NaX ziJx{MnE{8$xOJJ+LmE`6+AZMtQ6=tVl;SA1a6N3~@uD^tkxI$f!ha*P7vbcBV~#l` z-2T_>*KEB8*h!^dfNG8n=ONm7n8IL{D`vNfib}FG#|Q_p>qdyADbOf)ZhP??_+k7{ znLgT@lF)Kr7|932NtdAX`>xcxy*{w+Xdcy0X%!tK$)l^;nsEHFSuOv#&P&x=CgePuz;J1ue;e( zjg-ju76t@!`|*824hrRRp5_I?i>EPsbGZS!HIs}qQ{+YSa*>n^y{ znQ}RX+fiawJyU=B&FjM$=vwTa(6x|JL=sRY{>mJXG9H8zijJeR|L8pXc^Gfqncg0! zh1GB{J!m2;gF%f%1+fyk7U?$p$U@Nl;6y{UIppzeUL zaN^>UWeHe9?57kh*T_8w`!Oy!XMGws-`aAk2Kl=)g9M)JGhO{O>a@}sd|UkKc1U^~Nwo7x_>pC2fu;2`)hyJVHT2$e+RVr zeD3Ap(VoD0SJd$cXxKT2Iu9oQ%mFd_ICCOkrDmQiLOeDj=q$rnqvU_a;;~n#%vE|( z&uL?*f0V&fc$0&_`1?wfD~FRVH4KaB2YMAAXVe8=U7De{(<(C)AlU1A&h!|CHqrY- zl}p^9U4O6_Vw5hy+Ttyy*w< z>%dkom#m}W7(ode#lUU(B+D$5KERjX_O6bY3>RhxS^v|*Dvh1476 zT7U3#KQ*1RD&3BOg3)V#Lehk{?OY|FH*+kB5tO(gFG3y@e5Go~RJnjYUA#$AVfOpi z>UW#TtnG6C1nsH=gBJe~#yud6fomF@1NbOG#P^jVAwu}OE2Osj6zvmq2rE}Jw0UVb z44+$8L17P>BQXAmTTfbh=~!{521Jf`dMm9-b1`uA$>O$#CcQkP9z6H;ZWm};QRP;* zz)2LD0$uYB>H4`lq)SpHan^Y_aaR|@pt_m#bM(t>950JyUFw~7Ged>rl>A}NB3+}q zNJ+T6^Ihat+*+}tQK@jO#Rte`!xB{7hyr&D^Kcx4#taZ&j~)hu=skX1j-u)}Pdb}$ zL_WZ&k)GCGO8Wg(rHG^n!^Lcb`^#N#_DxwEYjAKqc=Z3kHHaLgx{?^1Zud<+`@)2> z+9JBzX~56OXo%!mPb~RUfDpq}aYATr_{w`Cj=6Jl97sVzCk# zb~A}dh2|}Y2j;HJ-b~YU^gY`uI9k#t6OO2MHZuWdNIb7(k~$J z#StHs!jNVl#yk+8d%7&BADk#Jlbn+98d_eJ+R|V~!q&Pb-%=T>+MXnXO0FDYr{wHf zdT88(_Xd0QNduwj)+#AD1q(>2c(^I#dH$?AS*Y36ja?z~JYpMUUb_Z7FkR82e&Gct z$;8${q_}5zomulK3Pt^P<&bVsu&?ni5Eg%l0A7Z+Odj9F|Kx&Wf=O$Fu<)U0p#KeW zNk7VVAw{=(Z7r)^gxH-dSlnmt`g&&SOD-PE44hwj5+qcCCR4Ll)39C z`jTPoN{7EZc#K2kg5PD^1N9S*;$ZLzbP}1!K#K7qSD!c*Nq#p)7_0L#TfdQnr`8f` z_1U+VK zh_tsgVB|cSs88)edJ^6g#Y{snR(`eKrr7+-d(cp4?Cs{=rT%VNN0%o?6vMXMRKY-l zH*&Un*PE54w1d7H_MPHW^cj1dS3TK*$ z{*u-mnri32(1<-LDA!!@0IQ`$v~lOiX^> zVT{NDh6xs-jp?`hwHlh3!d~lQ*RuUI44v&tTw{|h`$7qX$+qfXO$9C1+qI9#nPv-; zIEuFpT<%!$a6%-rplu9~n8A~iP**4^6%mlv+;h^wjse>CAiO)wGCTUIgPn zdCUBcrooiwA8Y?5XNwxmj0sP`?(y17doImi4S1ABxpQNFyzyEIiaa^4fTZ)Se66s1 z)W|BH)(ps6%=)f8mj85$V&;rs%vAC+^$tYE;RmM7C{r_~{-Y*pL`E|1L>H;0jE?Kj zFlYOb+xbU7#pcaI`$C@-sPZ?lxV3-Z$1qQQ6DgWCc*1NJLEi0GI8cA8dHw+yY>Lva zOS#!+7RI*ZisZ0Ib3Oe>Nhee+$A2SzE1CbYOrbh}#8Gi6DhZgo| zmUoQzsJ@}!o>j8L>QUKHv_VKhJ5$2=Z%pfWXQ9=#M>w8J!^|}h9Ex^((a~=Bm*EVJ zQnwI?QglWx2SGl65kzzg0iNi;R;>9lz&;bDwN_JFU#b~@+CutW+(k^6V$zSzW%#-0 zj;k??vF!3-KdenL#Vg=oOgqM~6b2_1l2J759Cto8%&iOt^4}o2YwTJzzGY59>trklV()`lw{EnW z%IRIH!${MD*kel#7yZH0dlm45NsP(cXTa*iZW!e8FEZkEHsjMP_kE$KGbHItE~w^t zuz0+T;p&+=R_k&BLL@=1+`-Y96V_cKZasIK%6^P_+gRnd-&K+D&Un zRcf(nAr#L;MWhfIM_(teoHL+te4>!5AXJ)5STlF*m0M%vS2IGfh3Bo8RQ z@s!OY2S5{JG?(7lIOPX@98NjV^#@~jUQz$hEaTh+w47G*faBIz4vWN&Z+md`<{m7J zw0uPQDZG?D4w2DFt{^n1-F)6}S4vxOf0{PH?AdNIv*RaLTZ*=XdrfAEjRxg~GUlvd zcnT^mDFtC9%)HEiBZV4mMIz0xxbroe?tLQ>*@cYn}Oq?nHy3344pq zlUZw`yqPh%R>q`;dCd==iqK{SXdFLIDX#wGzOGq^!1Al+M_Qok>7;SNc6k@qK z*8j-04}!$L@bgbs-}h zN}z5DYf-r6s6KlWW6+nx6x*I`#O5psiqm@|nIW?>o0&0q@Z2v9Jl1**%J*=RCAo-{D@s4_8A71d zzcD(VR1Plz(J|pYeoi*^<(M6#M3HKB%q^qu3b()+h$`SMX3})C%^&AAFN&}1|FyP0fi{NS5luXF zr8I%*5M+#Hj1tI$GP*A8PcfRm)7TPS4i`T3q0Fc?^0Xk41xHg~z$FJd#RFN6ax<(2 zmbZIa<^;EcH{q#zisL~>2*vtcwK?UUObE(wc-3qt4EZp*(pSVW@{E-+W0zLmU9}zO zZkIYWmu0KXwL@9+7#C9c^zpFKkmYP@WGFrOz;an&ieaBITW4J^fgFg7=jaOtMi;M0gB5>kH@E z<63&v^irvFp*(~x9x`joa3xjvV#ayT6DF3hqFG*0Njv@bUByd+-~mO>j$rW{&OpLv zJi&|%k*VuL*xPGKFas@2Dub9`cs(3u(SfaQC0Ugl#&ex9?cr2eEmv^HI@H*&^XQpy z)A&o$lVSFm+=m`z)(mz{f=#Y;(%P853lv3Q|AMYX{S3KAFz?Q8M!}=N>TK%E%c!~H z7Vk3B&+8|TmkVQcSKp5<*cu`ET7^Y9#LKA=|Yj z>Kw{x5N~_XxDlJlJKmX5hQPKgf8xiAa-NtYXNFPZInUtsE!gWZ&UT!1O2Aj+s-bu& zk{OWma%MU58Nc(LdQy#j`Txh)n}$ueaxL?gSI zsZ>bWm+T6aJ%q+KiL#8P2w@m9V;g3yGZ-`Tp3(Dsf4}#8UGF=8^ha0CoO7T1+~?jt zpPL6|bq;`0-6mL_x6)K`R+h4w--biOng?*rup&TEhBrN?dKP$uAD)<8yxy_7^bs_X z^DGWw<7labJ`!{ogN-IP1T}}Y0cXL0l@aj7qf$mdg+n>VSnx*;pH}L;7-e-G0+@=mRc22r*bx_(VLTZK6z9|*yeWY@FW`%L3#TTA8t zdnwwfEd5*g_5UHK;QZ^!%Isf`+Vj)u1{*-S72u7atD*n|48XxnpFtf4Rz|gaFgQttR`z`k^erpYvr!tSaW`SbK*6@jZT{;Wj~Q(`U?^9U`5#jk#vxe zcRmDz($4OHbJ5wq03@Q7fCzD%eJ#-uT==iyH`rgs4f}IEIG$xYDrOa(kbZvsxKXcs zhwQe8FD}!aEN1nG)-qh%3N7EeczR-(ZkqCv9U#?8Up6Zjx7NqDppVG;3|;F$SY2U`PZO_>cHzM}RXXw`~V|jG85Kvi~{?ntXxs zfd2XIwrZtlKuofJiCBLSyk{x2IgK~hari}}9^+L<;5jGW%b(~JYa49Guz@p;)iDFjh&tiB=Z`xawdr2)wVWfioNNrObXGe6)nA?jy=7e|rW zzk2e_y6J4->o7Ixu29f5Xj&Bi>(_0JGl-eGG^x89Ss?2r`)f5h^+l?%pTnAhF7H-hCfSJoEQru~0AB)x~z# zNO-tyw)=SB@mrGTf=xaPTdxb0{Rv6HTCtjqhT8q^f4)U>Wl=!1Tjq3vLha!=S~RaX zWxe4C?AD;xXYLB0D|IFXDN%C0r?4?aTL*`97X+jLz$lJ3(p#_PRsoPCA|CJ|7SRE- zlvd_#(#u09Yq1*nuRAO{2;x0jN_MB>8OpdA`**+9Jy1O|c5-}neScRXe?8(fDhh@M zhb-QUnsh>?vbZF61X5_0_~$=?Bw%@8z+G(#h&0wQU+@VQ0bh}=&)STz{NvvWH$aQE z9JM-!1%KBi)yED73InlzWDZXKsNql9-ePae&Mys?Wke;;mbKGen}kvO zE6P3G3x!>=D_XZCYx2{4HH+pI?DOP}VEuT618XgQE!8lEXcj(4b^UI=rMK<8Di6#2 z-3GT;vwoJel^oCn z!Xx>1c*buV)}F2|2u;iN=D9jys<>*M>&PDGBI(<~!B(K59Y`DV`Wt^cJn-jMS=H~M zNFsWDsxcPe!?e^isReEgacf~Y>M z1u+jTN}LnC5@~hbjf2NfUVlKTZHgGTZ?#r`f5!zt(jiwX<}X97`~Zj+er$0$uW(pG zCD%6A)^EQ?&oLtoy=&~f?#;^4s~9h9On?sPOmH$1ypAAD`U4D%as|Qm>F6D@X8K+y z^ydaGX;7pc>GgzKQY-aH^$~-CeBi*9*KpTP0M~v^tj)MIu&%PHQks|*N+1oUd#zKq!s&pz;IEQr$I*HOpXlX^i6OTz$Z#4S;dQp?f|U${gR!2OE+ z4YIwCWU8kYw6$XBVI}NV2h-$5UQiz;+rU|{=!wiccaJ$Z{b)#ds7nn zP0;n~l*{3n%ju~W_9@0V_{EScp5VnM@jjuhgK{to{Wbja|CPzrdnDLZ*6E33Xl!ng zV(EdeKHMU?*9#AeKNyxW#C)rXb$wDcy&08(mS9EIQ8++k*S^Hy@w$ z%R9(AT2PFg+*1908nPQA_%6$?P?0-1f47JmQG2&{Yf6g$*(nU*Y>%mc&md}_)7k6QW& z@{&gWJ-MIZnbEMaK&u;Sg11n5b(#X(L+?>xlXfe~B%jV^I_)`+zbmeu@R)4IE1l#8 zgAD7~kIc6-l4S_r>ud0#sq&!;S_;tmDURXVO7nwZYY#TxFm(#9qH^c~c$c;;emUn+ zM9QJEDArpkb*9eC$ZzBr?+KPN^eAL5xl*L?RZuNyaw9<$V{W>SB>jx`7LP#TJjn{_ zdVC2k8Cdn%zjW@t_L8uG7K>X8CyX<5+`-bqND5wBvV(jqb3nl<%8HrUb1-aK~ zLt;JXS;&i`t0x&24LXV9?UVGFPfJR@JBLf_4LuzLYUMl~SWT8f_&Q#Mo(K+cnG_VB z+OdwVFU2#=tQ#wUtUT~TX| z@A1)DcKD@9J3?YC6N(DkG3#6wk0|iRi?7hkk+{&IJwS9#{KCN7P8Pd0*wSnu z28364MW!;I7JP~5{WD2v!{chg2`QjX^pM4$gi3x>%bho_TWZ!h|N4%co=)79(g~>svPvh(i++~n?Bjt zxKxh`kkJ=|qSl{@ZW2RvKxE6vgh4Pmzdy@DM&^T}usW#ujnwx#md~)}NX?Z{OOaX| zQn<=Wv%0-ZDU#-hn(TyEt~?*bAIQ--RbD>{Us;Yl5nl~=RNmaG$IpFnQ3k?c@CyRr zeCrw!eF;qADbQrlh%-3VM6RZ5^U7@-8X6yk=X&Umc#=A7L@ z%p$$#7wtQ)m^}#sIa%w|q>s2owScS!LdDVBp>6mow+y2gCHGkh|Eo?eQx2=5qk4%6 z_|t$ZD;vueSL7)XS!&$Gj@ACKl|R7een)Nu&!mj zmA(?>fnx(cn<$@Q7z9wPq$hFK%TjpK*hG*^7KZ&9WYlP6J|`^jV* zxbBw1Mt|b=5Jfmxw@EX^H8Q2b8yy{aA(AeZo;hE6v?1;dGN|whW z8t)@_V-H3!^w8xlDcUoSvb9^>fvf5Y-sLS2Z_*F z%j{9;@PE8dIZqSXJ;pf=?;zjKTF7HkZy!0m3533b-hjWd)ZZ_wM zCZv|xpB%z2Kegv>q-@n^CrpTse|qUn%T@;wS-1!5u`N-O`=$2e6!vm63=)v*%q;sk zA^nHeO?iTYq|g)+iICNkv`1}Q;g^D8$e(-MNkpc3;t8Sj^wah7)C}pJD_Qie5V)j* z#HyVV#r{Iv9+ZzQ3{nnMD~=NcUYt8#V4-uSl>F?$!FDw!aMHHrAZoV|%Cdtw=jCe| znT!ud7%vUZ;M8bI69)%7be`j;^dG~Zy;;(ia)^huQnV68E6_n8GBzN2LeShhQKcLU z9Quj5=gQTh!9LKVbK5sB=_GX(7nA9KEsywVd?I;_h~LPQMD4VAz4_rO&f0p8)VwU^ zC@aFt)NBediHRMmq;l}1DU>g|?{A_+k)!2{^xSKW{kE3lXywq7uzXYofhG%kmo<@cHJ0h|b3}_V_=O4~q zyQdfXJMhwi=V<_)=~VzKpi8oe7^Er#9>_(_T?q+A)vA|B&;&AK^Bjuu;>X1Fe>q?d z(gXmP3cxgBgL>TrA0zToGN|<|_6cnm!Z(o^x;i7DI>Pcs7&qmavro3$(GWY#?*MMw zJP=!e!;s?|q zmG&WT6xw(fR*w_y3hGtpVhRN&DC8zep(C_>L<&Us7;WVF3)zn311o71f0(|<59@ra zf|GBe=e5;vL&An}<7wi0U`PLSlk@676**e+(2>m-VOvY4eZyOfC3DdHHr0L@RD556 zNbIh>ycc_n6Q|lR0nthw-Hn1C!^EAgKSQl<$dG@cmLhS-9<=dS$gbN;>||W$)lEqN zxuXCY&i%W?L0k1-F=$FRgsWXG%lJs>m>O~t|A;JHop9@+sGuQS zA3F7=r}70|63=r@uCR=<$)N#BA+Iw_?KP6Q(mr2BFvgimj>JkWl+S&1^M^bE)64TB=FGm9=pElK!}ID0al+95q(~SAn36n$6~eg& zKM+yz&sh+2%(WbAR%cc`AloK1>37hT^9_YpCnR}(RD;~Y4GW8J!X*{KRRY6=CU3yE zp!FG)>v$vKt%wtmbi(cm-ndLF8(LWS35AHJrMF@3igC>*VTRebEE6GVa}qSgxE|@c z$*>5U`F3DjQ}PHv4lHb}a^GuUPo23$(8Bjo@fqffXWN9;{DavM=_36?Zb)pys<*-_PV~4gd|i75)6#q+sMLauzxF5(qKzcTP)iVI`5k6MG?B=u`XmLAlKL>pI#)hgNVhi( zF2A3ZX)l^rFUKlahWt9qd;iaWD>lHV25}OD5-==>L+ewcZj(^b!UKpH2orPmwdy!Q z4z@C2kAHJsF)u3C1{NXd=kk#2wSM6B9#Q-7lQXi>XEhaHH|ezz-!7tad7<@=219?; zYdvOrT;P6s1Cz{f4yMgb&@LiH?-PYtA?`=WqkX*c55Lnj@r#eswiJ+~uUg0=Jm5}# z_$f=LnrUyCK&ZwrBDo=OYtp@t^Vadz7^Toz%KTSl$&SSUyYcn1z|GL=V=NoOB2h(_ zr+-s^0dJfJRE@uJGH3SV5p`$mLVii`-k<-w$$}SmL_W7`-jo{6=u`{Gl`BaMcULBF z2}j`Bo9!KP1adX#YQ#i%k4lGYgSn&Q_P42(;R$z@=cHQT1013~0C>>@tvb5rKpP5( z^IiS7{g3@pwO)|>IN!|dr`0G%)2?ku-bjx0-e<)@jJ^y-5&xrp_Z z*bDe(Q;}SO1kPAOjlxg&x7(UGk<6$m7_nDyS@b1ZLcyoB!<-oS%>74f>IZJ*civGG zrYa1wB>wGu!uVN5sfTv*;lB+R+AL}-ABTCs#VNZgA7f)rMq?GKrUK#D`d$8B7q_pA zzVK^#vd#%Fe?v{gDACQ&wk|KE^1>jtRubCwz2O-;x(d>~j@!Z|jaT2W%H5}{Y zSb4Ml>NjWmLC_EI2@u)E;0SuAz>NftrE&Vc+7^P|tIC%~%D_zr`BDyndAiw}uVlOGt*^>dY{yR>DJjb;N zAbQt0ZsEYa-3IL;SEQ-M^BWE)egTKg=HJ|))c`c#RWQ+|mEp9z0?IA_OUem-D~|TF zf@^NlNu`pVPrA78nyay3Z+NW7@NZ;Ww`IZ->&Y-br6&Vs7-w@l6!eZe*J z_K_uWab>{5OK!?GWOIS1r*)h|WUrHG(HGE$O+*)Y4r`&%SLVPP8eXzL{6CrVRmGu2 z7AbR(e5xQonfur{VNcEW2?#HY&ETrm)m2n^Y#0owc=0MjC4pqOvZ^t!?I*9Z1jmG8cxHC_KOxQqr13X;-k6_8mY zW6sBj1+rI%1k+qRJK52bxj~lMncw0;&2xY8?eXu3BHxo_we|9uD@R*9==#RIHY*dZ z!nvMlzgJ{gB@}YxrCx`|`@~H<{SUv<ZF?w;KMfn=wto!=5=(GfY+Z9a(-nmv)!jk#P zc|rNx2`;UOYB7qj%PyE`1+R7GY=io1zd4QTdo5tNy>pWnMH;$TQq0hjWYt4r&KGF~ zCg=Bd_y3^x(TjJ}vjI49ELQrIDWq&jM^4C3SXM=~wto4$P$|vcT8vVxKbkUrCvWER z@Uk6x9kBdAELnHd7o)t9z9&I__3?u=Xr|x*?{T#~zs3h!H}IBoE-HXTQ21|!G}bhn z2e*S7Z`zKaYHbYK3}-fK-->ypcMfSHbT1a6_$5@bWuW*uhYj_ zaH_6=M~Y=~KcRIOYWRM#E&b&(YDOyAn*s-lwbo2em~ZEG$E*dou)7V`+D z@S8_#L+8UMK~m?f6EY20De}SS_Ru_7SrH7E3wR?!JT&A^@wz=a7#Azu>#!FEi7KH7 zSr0)Zf_O@l0kAV~4wlofi~sQ|3@0gDbbR7<>z8D(hk|)M9w$*TAr@gJApD*UwhOQi z9P^Kai!P$QT&)Z~a5WLj>PrX~)~bLSzD!-m=?I51?A;SE|Q9w_8HF-VFfa{2GB%Jy|ncO(}T*%3_UoqCsXA67Iq1TTVJ3kOexS@KNr+nEx%jt{5o`Sfn_TUT5`v0}c$N*#~_ zK193!Hn^NJWdKV$>NZ#SIn$VVpXT-R!P11Ddx7D=&i@h6~%7 z$3>jV))+V_>i4p?VQ7!wvD0AhfwAv)etQofhtJ(pB+?G} z1RVh2){?*gma!m;^^I(7n2TWLmB6jX`&;650C+2149Ip_>Rx|}C!PUlB{NCYppY!N7l#sW@oS9l6bhNFNYg906Y;sj$GcE#N|)8cb5uipN$O>D5M1kov*S6rh9Aby-VQ62&gCK zoM3QOjF3V@(?sampz?Bb<-&DR>XP%z6k`jp3@Gor-ul(be)82&Z~x!fH|(TbLv8F5 z0(%yy6+_Qxhh9(~TnmsboxRq?H>#Zf>g*2;J)O|0Su zE9PW-GDtf6B&?GH+@i8< zNH!AXr7(29sPiX#km-!1Zz*b!w$s-CZJ!oK$YAJi+2>Bt(DNK(0OBKV(CJ;=WvcmWeJ9GV1-VG z#$g!9k#;$%zeZfCsck>~FD*ia=K}P<7<58&Wt$|y9I#yr1=Dwm2>u{>yv16Y7Hesc zg_*I{J|E~}mY)sg7BOIsSAMH}Mm!e$>ZL5!yqYB4HQje7UL?_20QP)t`8!!ZP61uU z#WdKjl>`_tT9j^81a^TqBoh8X2#kGsbojMq+}WGT#B*LzMk6fbz$aO^Gk3Fo zyaksWl0Lj8JEIqPb?0Egq>Yp3q$tBeT9O$IUP`f$wuZXdb1MF86*T@nEp9<{8Y5Fl zZf9R{V3wbI5l_=`>_G%n5eP5F1ItB{$2C+kf^2Focvix`DPSDTx0CG?FtE8$L05%7wvz$wj3$~x!OU&!ug8Ed-E zwcYXTX$hzO0G4wfz{hpaD+l+QS4F+(}FC>#lEIe8M;rmsQUYOma?e)RTkm)m+hz1__VZSW3fH}Ee;BoV;A0lQuenQ z7sasiBhU^~!E8dWIJoN_uSX$i<)8*p z0VVKO7v)nn+Mn>VON0W89;XC}ey{rF>_-LaVW%BTOSWlG_Nj+5hHzceFnB zJml_v{H!fd4u}S2E0daA?*&RGJNoG126O6eGeeFuu5B}7z_Yox1MW9Aj>U2t8Fb5u zBv}e0U(p|w7}t-+I@mE*NQo2PEGlQI?KP%;bfrBf zGtn4=xJnA4AFW(TwCSiOGZVNUCUh0Z%`H$nVMxQ8i^=Z^373J&Wn8*Rij_hS500uY z&i!ayF;zK6n@sF$B&bA5c7nNck#LXQe@|PmeNP%qv1pB#x1S1-il03IgZ3xcKauuQ zzeI!VA54)u_;55K@-uWi8NQeZ%j!|Ru%xs(&+ER9;0K3pZs-L!(3zDNV%-ohwMs1e zcjkE>0Nl*|ZSyrj=|E~A^I}~DRE}0gkKA6KFk}>0%~D&z;2Qk;C#?CVbJqdhj&7SE zBzm5`582Fo(}L_im;Do#CIz&s+i9yo1>>k+Qj6To)LIHCUtA0gZ2;e?ynYy`eQ#-?2@Q0rvgS%JhM zN92&@;ihtMAk}Ms>Bs|?;WOR6WB8&tmCto7 z@Ywe0XvqSA5rC7wbM;M+sIlGnIh4Oz1h0e*>xFvCYQZz)mYYo?etZ_I_MZ& zyaIC86!tCYH(FL8KqRSm4XviaZi{Y{q&g%GfwXR#@A3r=#_R!x=MFsZNQn276l2y4 zr=DmsGgdHz@4zwR>}zpCsScZt!MAe*ZF{9yPWYRioO2E z`_lg#kLYB4G7(%xM1bV_U5=N2A`b&MX)9~ucFLb!I1&|-=|hNXBF7Il1iBViO*hs^ zW|a=7o1fZoI@mgZiLF{vU%AM<1p9*7ccYBh8B;bTqPKBNM1qpM0AnU-QGfV!T06!( z@TO7myl}zsdZzqcQLIn*=z8c^7|uF4%a#OXq^uo;o|U1E>!7CASC}a@pKT*@bs&Zv zN3XDF{M@@~k-PQD-U|p2XMZ>+Fg1`cjNZsu+1?0a#)qwrBaf!Fjecag#;S+@vnQ2T zr*{KK8{nouD@>S2#`evCoUNDGpl9kHfbe3aF;dCpFi_>*wtSs_ommpI^O8ETi|&b6 zr}b{ul0ou~y-RTwyw$AhC9IZ^bTPDvdY`v^Wsitm6B=5PZ%KYDxmW0+4;8&pTZ9I+B&i~1wnM}rKzu_!*)Gdq6eupMFB2_n|M>c~-5u1-JM z35@hHr$ql3iE#XbhmbL+aH&`#`9p(-qPu7;D_SL@$cU<$|`>N(p7HZd~_U0Ph$m~X%p0xT5M z4fSD@`hsDrcZWwt5=y}Y&|W*l`RJB*dS&XL;Dw5|=g#~*bS9g%r5|Xd-1uqa68+BD z2aF5MIC3to<1955JY2WMK{j!26JcTC(03SvOX|ux6_PpLM|} z6)t1bIXPat{TTljA~!m24Z}ZXPFYj*QA>GqyPmMZ&Nt^?ivGCY8;*eR%RbKubF9p{|yPGJ^T@snD7^Po@O%OB)IV@Jnotj+R^ z-%ydYu#I2ub=2;=yFQN@Q00`iT!1XM@;$cqJhDT4Y4hlwM~CY9X8Beo(U z=5Sx11luOM$VJP(mau!*8G1s`Nb4jkBKnp`$T_~6TmKguwrR1@w$j)Jksq2L$&yh> z_C!_zCanGgWKPx#dM_fLwv&7u^Cot)CZC=P;TdT7+d~DEP~G?Qu0K2{ zAg{2-DcH(b$;Avll?bzaF#oG?GC{}ntuwYp+PaP85!u&deI81gM*KPz+a z>o6V+Gz25)q`z$kxr5n($VWxCU?A)14Hlo&A_u2dXk~r%Sx=eb%JDG@1B3M5;Uyd= zP7l>u+j<5Tp!u%Py21PX9h9BL>>6V}b;)`1d z+k8WfZ+bXNd`w2$)NlyUsWFksw9s)zCTkXH;<}BEo$y>5eE&xML@|T*5LZtZ zPIGv=n=K*Eob=wX*RB;1DMS~6snHcs5$nPCo$%Fvbig)ZT46YRhU5KV4WZ6;$E4C8 zJ+e=Fbb5U&7}tvyXg5+DEErqF^jMjBuk=}%!S71fa!*&JyL0+s0oSUQYE^0BiBNOu z8=>~Qjbuph4Gv4us8m`yF+00+aXmCt1y$XyFX*+N-+y5Ot>DDX#ulZhLQPBKG1Vux z|2XHEs;QED!a#!^Cd%K^*G})aEuk7WtSuufd}dERb0dPyt^@>8s4rq)xbp`AiJ=x3 z*LOWsBQ;oq|DS~^E+5Y@t?@?%e#W<#4nX~ub~=9#Qg0lI(`pQ9wI z^3e3;i3dk^B0ffuwvRweM-hUNVkepC@T7ubF}EIL=ZBs14u0vkV3)75-tuH0IW#6~ z9~Z*|g=S4Luc7~Z-C1fXcynBRJH8ny#CF=i&q+I?s4IF`(dAU+&vRwWuPaU~Ac~iL z^1PjXZDq0T{!g_D2mIM4q@r7|+kqF6e?_~)OI!C%WJy^=4sUOJF6noWZwf3cztKh@vInXeBWi`>g` zB{aMWRtY}<6BQE=u^8fXpF>+er3mea<-((qQ)NwC_gWDhR<=2H(=y=hVXVxK$9!Bm zc*QG`@0KLUpY>;$l7GJMD&-t0Q)%YVTfBN8{?)~H0R)@7j zy7NNq5ZTAq)3{JRBsMPmpG&(@bbQ;(j>{Pxmycz2Aj?0go1LQWQog-n)!4IJ^$`4g zNzDTXW04er#3_#)&TDEI2`W2#+Q2KzO87Y~&9m9dFBubES<7FIRi&{gY4l5f53MnBj1-*hzT(bs&W z|M4QchQBel4gk0XZe}`;VAslvfT{}{j?tO1kC2B(ayk%`e>fpjZ#@5!f7Q6}po6 zhOUE8P8)`6moBK`{dgEh%YB{sIDjqs_9nh5_{YM$T4lhi7b{ENwQd=c>+{~>T(JO| z;~H84L%Q8gegvi{uXhO_m_w+t#=$#>Mu1w#w_} z+HQ|i?eJ@Hw2xO(u99L?M+JMY?$^1pkoNP-ZCUCL&;xSG$fvi_X2i|3uaD4lUmoY`MNXVSuF1%3N1ZU;@~_MF)(N}d=M+| zok|n2 zX1W4pu0ur`$R{3G0kK$3DN{5h!ydfPTCA?`)KkMk?Ze%JS8m3p%04|$;fU>cCtvY7 z;NsTfs2&g@laByrj`hLo==ueL!PMXXM2n>#i`AyQ#*N&uRq_UX4`{$mL66ePWrlT6 zRKcL!VS z+6dH9_5H)84Z@tAk1jjN6+|LfHrX{vu= z^EzK2lA<)XFht$!Hpd&%m!8n=rXPEB81Rw(tWPLex#Lz)j;UavCS!NASny4d*N!oL zuSs3KAX2q31xNtyZG;*gS=;#c%h`B<^Lb6X0*3Rw>v@{mW=ve@*kmZ#W~CP(jW`;` zTf9y2jH*7*MXF=8t=V_4(5qoI$o@~PM0O#Mt8gsJ(rH}zidwh`%LuSOxw}9LphzkT zt8V<&#O+mv;HGEa@Q84e`>AcJaWS=od42auZA7ojyU^0Ji$Pty63xoxkpI1$Rgi*+ z7tJgnM^6_<;L}+<&9GglsOknp*q;5+9LA$h7kCH!vgk^#!n`~7XXm$1fV$E(ZNT|l z=H)-wK;1LgU0cQ^L9VH9+&YCSpA8QcO)w*V@jwomT|xT(WZDUzoeKJ&A?R!qXAG2g zv*av)i_}||;gd3#R#HlC3;B{>cUT7$SzsZb=cVx_`?^Zpmo=~QfY~5cpJOG&f(?*3}x*LiA;^* zxxj+PntOnDJ9z2-ZKE6|w^xjvVFBXnK5lj9gVZ5G+IyPjIF2s5za#;>9eJB3lCykR zuUYhX*UWB+)lI0%Yj{m}t2s=o&feVTrXy(WXru?3ak!I{Qsf`t8{JF&2AP?AtB&+B zV!YJa7R5oe@9!raqz7_CromwQf;Dm2)zojxI3K6#fq2VHoA=vfCskxxK$VWzJnUUo zlixp@ptlkDLXjQcMiJ*rR`gg+u=7Mtz+}6P^H6tc#nK6xYfBAXmHjh(Q z6v$Kj|CeHXX;)edBQo+RA64%m^*vkzQ5Xh1S2$+ka5TlLNx znD+?ImAf9Xedl&JzcRKwWOn(#bsv8e|8kF>m8cPc;lUAn9Oa#|zE9grUF|0fxu%HI zz>FMeX1Utu#bi7)D%Gm?SX}Q?Ik4N9iwCI~CEx#Zc`t(7oe$Yg^N<`L4Pq6MLK+ zDq8ofJ>q~mC&+Ytsl0!Eu!{ZXdy{?+tOICOr;891pa3c=LBo_cihzoUO7$g&(516>jo8WwW; zf=%pIa^_;kvOz*I7yKua8Gu3q+t5EonLT+9QBrdORwTN9gt=Pj=Vp1!ElP~3r#@5T zoWC)yTVsZkTU1M~N+i#2@^+>DlR|5XtUd9b6jHgzD^0KG1k~Cg+!b9xX`6H?_qBVwV-MyH{F~ zyp9f)egYf!G^UR~Oe5`o z0OG$)azT#$ULa<2&ZJLe3_D&sVR%T2nkUi~V$ri6V|IDdDyR*yV4}N}u^3uj3%MYd z)OJYcepU8%9k#FO?tJKXTKu~ci7Dnqby4Gne2Mik30#|>srh3mfoCiroeMe=W6QDs zq}&`$Dy^8{{WT_6`@8p5yXdcfPd4E(FvT~|y#qwOc(l9QYazY6Zg4|Dn64}}X*<-M zw%JeIirqWU;@IC(srSF}uw3QP8%vsVljgRtfU7U1=coHSd{Hnm6U)o>`;iq+C^I>6 z(GH~0v(k{|3R8=1!r7=s^&pdT5r58aFghsJTJPMNLtXs?`g8W_SeqGS65SgrN5Z@w z+K2Xb^r)}V`jVXcR*dY_o8iAiGF1^ zRshN;M#(;KT!*ZsifH+Jh46z?|g;K1GAVWV+HsYAY%V6D4!jenU7vw?er)w?w->kjr>RI&)FV7d5yg*vWpw$zkSQ= zH$}#b6#6nXYdNM0Ub}XUAP<~y&SBUTDku2U56`%nF!~Kus&hTn`N@hARQ&<% zwKa)2c9^%baH48g32rg|J>8br?Ch4Y?5`de`tOhLW0mM-h4@hr8L4hBMlM@*lugF% zgNGa`QnB{0UQOtnPNIpZ@V*@2%+K!F(jA+X2$pOU#?B01(ei`T=y?6nGS=u($XOc& zgH_wK+jPadYj(G($otu8M<}(q)I|OJu0L8~8E9Fdb0`RZeiIkZfz6ZWmXDuwW^GXm z2C3jSm8Z6uly2tW{Ve(dR}{wztEXp9P-R!~O^Xr9PhpS;8XS>lSU&^GcB6h9hH0{p zBfn$aNlHCld@L)?`Vd%N#LaUhFkDMR6U9PfK>Jj&%9Bw^WA-PS0p0Wo_m@~T-g%zG z|A5vc!Rp?4mpIy`w@({ieWreoqUO_74M}zq{tZ}jXv`hiw#<->Z=G}x3&%`s&WWi^ zHELT+27lTxbE6fYI0^kB8jc(n4Ja$VN6)_XhN1SRa{7gi5Me8zH8^(#5Hk$$zg1h=?{+ z9EFHFYO=MB&FXleal|YAU>CcqO>*Dl$Ma%-7WZdGfZUrRb{yX({oh`aSdJjrOWIxS zl$gR+BC9vmB!Anagyjs+c`s^N%nDa?dwZE&hhI^9DPgiI^sp?>VvXQQ4+poM24LS2EqBY7dr>0b zb;X?M1%o^D=k7sl2;%%p5e|pxo^|asKM*Qzp+8Ia$bR?Wv zE}G?@HpE*kH5a#}0S6g!K2bBWXP(62HeX&1nC;^os(8;W7v^ZV4#IiE%BZLuQX({dAP#$JujoSkl`x1))&bQ*nld^&WfU`$)$eJ#ghSQdZl=uA(ik%}Dz z21yhS?1PzD>l6|tUty_SW5X3A7tW4O$2{@1>(dkuad)M?Oo7&U3ohR;wZtfufhX(g zXepEbQsx)Kf4vgjLoA!amU$XMQU9wzY_jS*52DonI4-)X|JPr9QRb^ng2GOvVpWW- z6Wn*nzWgnrl=|W+rZ}Z_T_uK@V(b0r)Iu*lw`#*e)+)` zOtf9bDctYBp0Odt#x_PeAA{6(DqtRT1#=ud-xXaspfNiz>Eu!@W3qmz`G>oGR>=(`fIyPL)Z$<2m`C-<&J5XO9JuAEoau7eg@H z)5K=mNC-f|&-YiA!EgeIV#9*K`;4*ueo_Bi3JsT=^S>po(YXS7rpO<^H_1JI^q!=} z;mx<)H6eX>h0n}wn;5FhKd^wIe~a$N7Q7}#74Mu6>u6G9a(@Nu_VgSz79iM@KTr{5 zZFiC7)CV8lV?`SSbkBwK9uW`z;|7mOZ8%d^k`i%B|GrATZ(3_K?o?-Q0ODPN;Cduh zLEt|zv8leeTe$i*-V1aJxvPj8TrbwGKO$A)e}l~4Ck)B7aPApdt=EKo@A4gjd-&(q zuA2_W#4BBpVLSwga7*0J#L^#r>~4ffk3q)50}gj_A_esHjhWUsB5nXbDXz3X(0Ej< zMVIOqv=IK||7q_#qncW~ZV^2Kibg@D3Q85}MS4*LkuIPVsZxR%I?@SEq)3SZ0-;I~ z)I$*hq4y>bgA{3j07B>?)KEg;?r`4se0Pld_x`zK-2E>FvY)k|veuky&9%uY{u#5M zYhppZf3OmaRq&cL+Vxg`(q!Z-Xu6^`YDjSDu_?U7Bbrm-Ejd-#P_g7)@EP}#IEO;p zJH7XiOLsZXb;r9AIZV(^UG9BuUPrFPyh|!^n(+I+j*jYmqO$#D-FvkUs20}8mgeD( z0W@k))uBNI?;2ugRl7QX>CiF1nf0b99N+0!S(bmiWVlh9IqN8miz_L6JjfB%L0FVB z*i9@giC=Z&3x_@%K^|uU!YMobi#?#P0}jk<-9%NW#jt-KAy6&AXjcX!s?O;+GhsWC zSOv<#7jfsveJIoyyz*bu0yFT28rWs!)0p^L&-zGVF$;@C%B$JEYyey9GWGk}^1XVb5^6T{no<15GLbl>Wx41!NYG@Pd%`! zZnLJTc4V(GJ85r(2p%m=B-5*08DATkFmQtXl@s|PYC(^FJ3!3X0Y3&0jn=hNc3R;J z;M6_0Uiz5Ja}Jy2V{T`_;u_+F*)Ldkb(*#(Tg-!AUF6qW*PrOvnI5bZbQ2ue9!hR> zVGG;peFnoGyX9m?kHR;IRbJA!Y<122%BDSI$zHP#O^L|v+hCRLALQ3AEaM^c#5ZwKQox?GoPWQV=-0` zAD;vV$P_@UlpU>nDOS~Ma+c*Q);GJ+i>I-$w`VOUB`O#!aWU9!BhC3Tui=+Wl=3pg zlT%cvW3zGFxO5O%J>^jw0cO`nGF|hf>*IHr(!u9f<0(Cu_Afo|bcd%)gGmGG)z#P( zS5et~=_iG=oLqm9*KfZas@}Aj^6%!%%u-D0(~MZ>icb6yJOv6#&bZOAJn?LXESY%7 z=j0nWB>BA?6TReXI6PMXx|s83@vJ2J4M|=g_Jw6>6|SJRo$pp?Q3Cg~Kq15RRb@UG zhyiBnIPve2KwRCy*isNSP|v3+*ZgwotoZV!)lp1ymetZ_zM)`h+`VUnx{|pHY?FC_ zkuOf?FnxK|R82w5w9IKivXyM`I)sx|#f!@=$N1d8LC)uPE^jU{JY#%pZm;v*`)GK~ z9DL;r&=@wMRmAR5n^VFky0vCYj!1FCyNZnWZX2ocv!Cg45F@b&a=&3fujPEdpAm=30*s+rzJ@rm!O*{XrdG+<0Q#V^-O`wzDVikmg=F>p%hzE0P}&zRFt7?heK}1$A4Z}`v9DIOELhSB z+r)feNA6Z(z5}k;?Cm1mfLN;CY=)KSbwN|et*dkO$(}?fbxJDH>18VKY}n10hJu_a zXX97p>g+7G&=kd?Ab0iu*j9k-VweO$d(?gCr8?}8WaJ`)z@G4_uUdXpaXm%~@lIfy zY}{l273N|2S93qZT!5yb^71O*xU$}fOR3t;QW*K5<@sFeV}!6asqa_U+$x$FoJ#v+ z%pDaCg$?6h8PXsYi?BC0K4r2_w=%{pZ)sV==z5_(8DSOQgY@T>d0b)O93cC$2lKoG zdBKe`!bdlg7%Z*Z9r-dZ6hk=Uz7o=m7E|Wp-Ap);61~5Qu2cNE)5~W0dtWk3?hEiX z&0iXs!?ZsSb8MR2xJ)XvMvf&aHw%A|HO+td*jOj>Yu8ug&2!%nLqPm8CQF5c; z!;BiTc-q{hVY^7>654;kIBkuH2Ih+b*RQDTOfxcYNo$ImSfF}8?ijizd@8p|joJj9 zY)*k{YPFc}o9*F6yf2SDFlpRBou{qy>?>0B59@j=U;_?JFZ*XX9Sq&H%h~lKrL%Qw z>Qs|w=qG!wT6ifvtgpzvk3o4SW zn8Ye2)K{;*&8})ekU7i8F+vw5y!_7;%Cmpx51az7%_ens58DSP?h5c#@CL_&V`V?#q6wh@e82#;Z^agK65PHQqd-gnleuP%R%W|_ zzJVk!#qTAWa=M(=9A5p_T)a>9JQ4*;OB)O)miG!UO-`PZnv_koW8E&9mMwJ`@V&gX znH;`v$5mTD!gf}DF2EQ%ONi;SRcm(2wRy~T(2d?Vi^G9?eQ+-nRJuKVEJP3g0GJ9= z1C#J-HD+~yyP_n3_;DzUu2tQzIX1B;p{D6e#B&$chZK znA;Psv46U+K#h^vE#J#r3u>l*?6EglJ8NvLn!5-Ejg1*pR326q3mPg2Rn2Jn*|LAuN z4^NQnYGp}T?P&}~j7~m3C+=~}-FWxuK05yhi!l-yHSYtZ;qeAbzFN&7hEX2N>|lAOcM?AdIDs`Ubj!_=w`|xbBKvr z2;ww|Avu-ltN4Hq!m?yLo-O)pdiOzO%Lup-JMf!9#B+WXs^vxtCG2SaNmYgib8&9ta;IL zzXqaS?iGT(_Q_L6DebW!0^q(i(l`hjrYi|7h9ivfi9^E%!^)S+hkv^V7R~RB+vb81v&~{J%6ZLQlIVYq9VKE=Geq7g$P+ezfU_J9Mp#>WnaMd*#8(S z!kvaTr_*IdXdB!?4M3>j|J^;P+;32p;Ml+66O}BG#Xv>1_OF?#GI@UnOadv_fIj_y zF>K(Uz50iC{bv?ZeFFdg5C6XrD3js;>P*lA>-$toJGeNg8%KJ)dF?B*A7X^=^YMLX zOl~ei>h%)WvrnoOuYI4ILJXRj@9qVjc1#|qX*&(e2^mbQmg63di?7gmeLmx>$YSLe z99Gs@q1px0RQ%bEh`j+_bMy!;>GNos&vDXw{+A$(Wm+l4M^Tw-ARx(<(QI~{ z{gDf)u%rn&I-oG5TM1p)PfLI2_ldstf}3bwXC@=2p|rhw35pKp=VFhKg(4CkLj35= z7Pki#8hPK;fS(%Yh^1N*m`A9m*_0O-hhxu@J1LDgKx+?`CIGO!g}pHNJ}eq&z^kWp z$NWv~an!qxuu10^2L{jP@62sOo!G)^EAB$F5I4>+Gh}@rK)B*u1`%s^8FH74xR^0i zJT$e_FI8_6d#^5E`nG;)sFxfbd|kx0J{}OwpXhW}Mbr%>dG*Z@nX*Gg4oQeBu14Fo zmJ|H_6O6NMq=*7Z{WdyxKO#9ZKmgJF*w3UzhZQ}0)TGdC@j^wH z4*Gq5TsuDhcT(6%k+`JFl~c}pBJZ8!`Ol#ehq3f{S8hzqSn0Lv8)+eOuu7 z!C3XLK+gf(dw$J#5T;qU@H3t}r5}o%V{HbebKVv}<5Ow(^mRbrwVF%VDRAx+h%K(9 zA2pxM)vfeDKhjYJ`8CYf({C0%q z>F$E!9WVf{PWjBVAw}@(XDjaQ&t_hHI4qw;>C@dlzx%`CMH@mK)hRW5_|enK=g`uL zW+ccQGA#^1fE7%E0~8@tyJNgJD>Bw^xaqHft>7mCjrC_qHnuzOvuB(33E7n4ykbH~ zny*zC-xzbf=AwX{vj*DgIQok{PmBVeLZ@CAt%pI^a%nEYXsCPoeB8eC7U_FagBAyW zA*Yj$(f%iv(n1K-j$ahWUW-U35|9xkcnYVzv-p`Y$fjbN7)Z{dS6M)K+4@ppxYjHCi%=*P_|euUG-d#`xCH0R@kp=i;O{-k_NMeFTrEO6I6~@14NQ;=Kx- zA8xM3tsm5yO+XsffX?(vnWU(83&_U;J2qHl?e6QubqGqCDi&{{RwRvwU4=|;y_I}1ENb>5)OcPps&n1@a()#(;NK)e;ak7s#Q(s-B)8J2Chk_(*s`}7E7Yo8qMI`0;q8@0@g z|2-n?sV(*({oq(`(L%ORxhzJ%KQUCd`_*cJb5Wh;IJPh^O>?b~=%IH&$Vxgc9}nB+ z(XqZ79Q{-+jz)PKJt{}Kx7gjX0dh3BDnx=uJvQThGC~(@^Q%>V^5B(im3L8Q7E2J5 zHw7;)xRnHGsC^rsO37Al(mzE}`JcP0SzLaPXH_A+*ERPWoxmdU_xk%DM%trFq9V3F zjVw9{x|$=yuKr06yeR+cQsf$T_29}B)mRU^2+D(fVZ-1Vm~JCgiFQz!3C#p>C45+rflp5 z;Qvg@P$M2l{c$zZMNG3I%_MqgRP$-q`x{!#_e(Ee0^54j-NC~R6}t?3aExd^SJ0Xh z3=_MtSERH^z4#M56UVN-KmNKW_Oos$PYVL^JDdO$3^AWVcP&}yD%A7pQ?Fgk2#Tz} z$&|z?Vbt#m&!8$=+XIgj2>+pYevf0GVIyf_bZuK+rS~s(#MLD9ovE;#8|t6TNs1~! zi?`PXr_V8W>Xb5<7S*J&jQ&(;Viv;BS@;bauE~?s=Vk73VCLmzm@)KDlVf~1M?1Sg zV-^fmGN9-*fU3VUE{`}nI1z+u#{Jk5JbKCnu4RtTXr@JbL(ac`XFsrC)uW%vHO z=(bz^W}4=ok8M&rt^5Ub#u=$c!OxrC3;*|`opYodW+bh`6;=w z&F!sNaW8DDV6}0Z|NE2dO=lB=tRvVY%CmD;*+R)8fxCge>$KbJw|w_(cqO|~1g2Y; zzaYtPFwyXSuo0$Q7wh$={AYAwyvqVq8rHpG<9zms><{uINkZPWkx1Y^-dXi>(zkJ4eB}{`i`$oMZC}7TIS!EgYIjPX%+E zq$W!*Xoj|vtH_G5X1lu<{Z9Q9%h5-ax@}%fj*VP~%;+3T@x0E;JUsJ4@A8$tOi0il zm+r>0(8&Nk&{nS#SRiR*Oz`IzUjuB`LP~?BAHrFVAbxB((9PhT$P6 z(Egb~wEgEi2hLN0(uXn;3O`$<6N^D5u@d%&PaQe1+b9#}oca@CjG}6b-W}*5FNdY-cmLpsFgYW03 zRg!3d1==$ro0;%K4HD^|aaZun^Z-khg4uvLnr0sPpu z2kSEF{!E{Fvds3w8OI)dK~$X#<`|`QzU~^(-n!#~xkPsP_6Bq%a6r?uRol;iU3scm zX8ZLK{EqH=ptn3?A=rW;bH3hgLiZgM#&J$Zy$iFh8K{T~H37PVH@1^-uBbY>H8%WtX}p#Fuq{`e;61 z-^R)~HZSZ(0a<)9KJVG$o*eM%TFp_$$kPI(W3%Q$#(4OVxN|8-_l^$Ei8m^CWv*Ih zLxS=ke>3bg6=0(KN|JEeo1W7Z%HH6Q^2i=?3 zW)976UJrtH9k6D;^H3t4hFC*1T4*>oMZ#;ky-7-cX>=2g`ThsVc8JLXuk-8L>m=XZ z+illOn&FwY45n60Qgb>uXU#UU`gs{HG@&P=8{&$b>cU%RdxEN?gb?P6Lj7;jVkngc zj+)d6hKWVWuru|V3XDT%bw1P!gYZNy=>wQN(Vvikur4R7`$_jV(0;TYoWHjY)vSI% z4OG08UagDNUF@K+i%XSHF2gP|Sf9<0u0O^#bn*WTw4MI<=^t*-j?Zo9IRoX@+wU=7 z6q1sHd>?j8*P?;pMJ@n)W{`Ah_r~LibzsKS;3be@K(o#1mkXB3o<87NJ*CXYpdK%S zO*(%0LnYDk%^1{kT88SW8q5N9hm}~O#nU+~qh6>tv@x(KM~buF*fOy$maEVb#nXyp zxh#kIsdJ!Vd8^&mC8AHZ@}{m|wZ85OO0?{&z4r9>8vH({oHy5LLT|VUlUA}l?(=sC z^kj$1r^;K^6@tv+gKH@g4VNa|qy9LxiZ9iEWA=sE} zdY0amBYmqm=9)AC2%7nnTAV37N$SQ2s;Ir&nf7dLWc;uP-&ZyU8zAsV`?c$`z39ucG7nde3D~P%C(s5n3%p+ReREgXU9fX z&z!=mH}gMFy`a+>Gtg14JL$L`Ko9zFzy`z*z*2#$1sEq1E}?&v3z%gtM&#?rg#KGc#j^oyCZv_ z(nS$>LnnvpuqT8T>7ogz4J9j-vytYI&)_6>8Lzcgd>NNVf5=}*bGwz0BcJj`4Xw9d zU*sm!uWgZW!B(zG2M<$ZSn5l^sTplT$bcQ>TvCzhh0W~g^J=w9CxyZZNL0ly=u%FK6~LLR zBL#GWsyQkZodxh&w-2i!1-}m(GB2awWgLjR%GYM-CA`5OGz8f#%7rNSGuXgt<|Q** z07wQ251X(0C@G0!xZCU=uE-(f2kn$;5_bypAlWE@3P(1E4)(YQSC=c8S9`od8%FoxtKeHQ6VOLKUH;FH?#YI@ zDv5$vuL>_cB*|38y-T&u9;Ywqw*)%ENQob{47wk~RhqvN{yFjrbL}!}`WJSYIpSQX zE>NHjU}10hf;i2_R1R(m2!udhTcUmc!J2#5sGbe<=dV2iB|~__IXXHmT}j#=eZWe;ipsgu z=JHB!=@9PNMpt*c^oTL2A4pE_uS`ulV{#iQdN z6~wvSkU2Ax2Mn2~XW|p7nE58(_=^{delhTF)`p0+$=OcsiW2T&xecFu{y^fW^;~W6 zoVME&Snk3xLu51|L7C5bp|CN9q>KMVdcm%KVo0Z%F)-F~+mHTj`K9{!WshwYf!a0FK#Sk*KEy3(rR(OtTPw*}Fsy0H*e-F@1~Smg%UdxCZW(3&Sxg<8f7 zRTp#MJY$nx#!u@F9^qsfo((qmGCv3TnuA}WBuk-Rjk>lh4!ov%cPi?`SeOJA z9muZT9nyAggQPz~x&D&`Zh8&)3e~$a09f Date: Thu, 31 Mar 2022 20:55:12 +0800 Subject: [PATCH 04/11] fix readme --- configs/distill/rkd/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/configs/distill/rkd/README.md b/configs/distill/rkd/README.md index 84de61ef6..8ce6369f8 100644 --- a/configs/distill/rkd/README.md +++ b/configs/distill/rkd/README.md @@ -1,4 +1,4 @@ -# WSLD +# RKD @@ -33,10 +33,11 @@ teachers' performance, achieving the state of the arts on standard benchmark dat ## Citation ```latex -@inproceedings{zhou2021wsl, - title={Rethinking soft labels for knowledge distillation: a bias-variance tradeoff perspective}, - author={Helong, Zhou and Liangchen, Song and Jiajie, Chen and Ye, Zhou and Guoli, Wang and Junsong, Yuan and Qian Zhang}, - booktitle = {International Conference on Learning Representations (ICLR)}, - year={2021} +@inproceedings{park2019relational, + title={Relational knowledge distillation}, + author={Park, Wonpyo and Kim, Dongju and Lu, Yan and Cho, Minsu}, + booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + pages={3967--3976}, + year={2019} } ``` From 052ee3b33b787fd06e36f5d58a24aca27015498a Mon Sep 17 00:00:00 2001 From: caoweihan Date: Fri, 1 Apr 2022 14:21:01 +0800 Subject: [PATCH 05/11] fix rkd --- .../rkd_neck_resnet34_resnet18_8xb32_in1k.py | 7 +- ...kd_neck_resnet50_vgg11bn_8xb16_cifar100.py | 7 +- mmrazor/models/losses/rkd.py | 103 ++++++++++-------- .../test_algorithms/test_algorithm.py | 4 +- 4 files changed, 72 insertions(+), 49 deletions(-) diff --git a/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py b/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py index 0fddb6134..d64b25a5c 100644 --- a/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py +++ b/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py @@ -23,8 +23,13 @@ )) # teacher settings + +# FIXME: replace it with your own path +teacher_ckpt = 'path/to/your/checkpoint.pth' + teacher = dict( type='mmcls.ImageClassifier', + init_cfg=dict(type='Pretrained', checkpoint=teacher_ckpt), backbone=dict( type='ResNet', depth=34, @@ -64,7 +69,7 @@ name='loss_rkd', loss_weight_d=25.0, loss_weight_a=50.0, - l2_norm=True) + with_l2_norm=True) ]) ]), ) diff --git a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py index 6b6d2a64b..9ecc8db40 100644 --- a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py +++ b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py @@ -18,8 +18,13 @@ )) # teacher settings + +# FIXME: replace it with your own path +teacher_ckpt = 'path/to/your/checkpoint.pth' + teacher = dict( type='mmcls.ImageClassifier', + init_cfg=dict(type='Pretrained', checkpoint=teacher_ckpt), backbone=dict( type='ResNet_CIFAR', depth=50, @@ -58,7 +63,7 @@ name='loss_rkd', loss_weight_d=25.0, loss_weight_a=50.0, - l2_norm=True) + with_l2_norm=True) ]) ]), ) diff --git a/mmrazor/models/losses/rkd.py b/mmrazor/models/losses/rkd.py index db6873ee1..d20fe9eb2 100644 --- a/mmrazor/models/losses/rkd.py +++ b/mmrazor/models/losses/rkd.py @@ -6,6 +6,46 @@ from ..builder import LOSSES +def euclidean_distance(pred, squared=False, eps=1e-12): + """Calculate the Euclidean distance between the two examples in the output + representation space. + + Args: + pred (torch.Tensor): The prediction of the teacher or student with + shape (N, C). + squared (bool): Whether to calculate the squared Euclidean + distance. Defaults to False. + eps (float): The minimum Euclidean distance between the two + examples. Defaults to 1e-12. + """ + pred_square = pred.pow(2).sum(dim=-1) # (N, ) + prod = torch.mm(pred, pred.t()) # (N, N) + distance = (pred_square.unsqueeze(1) + pred_square.unsqueeze(0) - + 2 * prod).clamp(min=eps) # (N, N) + + if not squared: + distance = distance.sqrt() + + distance = distance.clone() + distance[range(len(prod)), range(len(prod))] = 0 + return distance + + +def angle(pred): + """Calculate the angle-wise relational potential which measures the angle + formed by the three examples in the output representation space. + + Args: + pred (torch.Tensor): The prediction of the teacher or student with + shape (N, C). + """ + pred_vec = pred.unsqueeze(0) - pred.unsqueeze(1) # (N, N, C) + norm_pred_vec = F.normalize(pred_vec, p=2, dim=2) + angle = torch.bmm(norm_pred_vec, + norm_pred_vec.transpose(1, 2)).view(-1) # (N*N*N, ) + return angle + + @LOSSES.register_module() class RelationalKD(nn.Module): """PyTorch version of `Relational Knowledge Distillation. @@ -16,67 +56,36 @@ class RelationalKD(nn.Module): Defaults to 25.0. loss_weight_a (float): Weight of angle-wise distillation loss. Defaults to 50.0. - l2_norm (bool): Whether to normalize the model predictions before + with_l2_norm (bool): Whether to normalize the model predictions before calculating the loss. Defaults to True. """ - def __init__(self, loss_weight_d=25.0, loss_weight_a=50.0, l2_norm=True): + def __init__(self, + loss_weight_d=25.0, + loss_weight_a=50.0, + with_l2_norm=True): super(RelationalKD, self).__init__() self.loss_weight_d = loss_weight_d self.loss_weight_a = loss_weight_a - self.l2_norm = l2_norm - - def euclidean_distance(self, pred, squared=False, eps=1e-12): - """Calculate the Euclidean distance between the two examples in the - output representation space. - - Args: - pred (torch.Tensor): The prediction of the teacher or student with - shape (N, C). - squared (bool): Whether to calculate the squared Euclidean - distance. Defaults to False. - eps (float): The minimum Euclidean distance between the two - examples. Defaults to 1e-12. - """ - pred_square = pred.pow(2).sum(dim=-1) - prod = torch.mm(pred, pred.t()) - distance = (pred_square.unsqueeze(1) + pred_square.unsqueeze(0) - - 2 * prod).clamp(min=eps) - - if not squared: - distance = distance.sqrt() - - distance = distance.clone() - distance[range(len(prod)), range(len(prod))] = 0 - return distance + self.with_l2_norm = with_l2_norm def distance_loss(self, preds_S, preds_T): """Calculate distance-wise distillation loss.""" - d_T = self.euclidean_distance(preds_T, squared=False) + d_T = euclidean_distance(preds_T, squared=False) # mean_d_T is a normalization factor for distance mean_d_T = d_T[d_T > 0].mean() d_T = d_T / mean_d_T - d_S = self.euclidean_distance(preds_S, squared=False) + d_S = euclidean_distance(preds_S, squared=False) mean_d_S = d_S[d_S > 0].mean() d_S = d_S / mean_d_S return F.smooth_l1_loss(d_S, d_T) - def angle(self, pred): - """Calculate the angle-wise relational potential which measures the - angle formed by the three examples in the output representation - space.""" - pred_vec = pred.unsqueeze(0) - pred.unsqueeze(1) # (n, n, c) - norm_pred_vec = F.normalize(pred_vec, p=2, dim=2) - angle = torch.bmm(norm_pred_vec, norm_pred_vec.transpose(1, - 2)).view(-1) - return angle - def angle_loss(self, preds_S, preds_T): """Calculate the angle-wise distillation loss.""" - angle_T = self.angle(preds_T) - angle_S = self.angle(preds_S) + angle_T = angle(preds_T) + angle_S = angle(preds_S) return F.smooth_l1_loss(angle_S, angle_T) def forward(self, preds_S, preds_T): @@ -92,10 +101,14 @@ def forward(self, preds_S, preds_T): """ preds_S = preds_S.view(preds_S.shape[0], -1) preds_T = preds_T.view(preds_T.shape[0], -1) - if self.l2_norm: + if self.with_l2_norm: preds_S = F.normalize(preds_S, p=2, dim=-1) preds_T = F.normalize(preds_T, p=2, dim=-1) - loss_d = self.distance_loss(preds_S, preds_T) - loss_a = self.angle_loss(preds_S, preds_T) - loss = self.loss_weight_d * loss_d + self.loss_weight_a * loss_a + + loss = 0. + if self.loss_weight_d > 0: + loss += self.distance_loss(preds_S, preds_T) * self.loss_weight_d + if self.loss_weight_a > 0: + loss += self.angle_loss(preds_S, preds_T) * self.loss_weight_a + return loss diff --git a/tests/test_models/test_algorithms/test_algorithm.py b/tests/test_models/test_algorithms/test_algorithm.py index a42886e15..bc2ca6dd8 100644 --- a/tests/test_models/test_algorithms/test_algorithm.py +++ b/tests/test_models/test_algorithms/test_algorithm.py @@ -497,7 +497,7 @@ def test_rkd(): name='loss_rkd', loss_weight_d=25.0, loss_weight_a=50.0, - l2_norm=True) + with_l2_norm=True) ]) ]), ) @@ -527,7 +527,7 @@ def test_rkd(): name='loss_rkd', loss_weight_d=25.0, loss_weight_a=50.0, - l2_norm=False) + with_l2_norm=False) ]) ] From 4e5551642a66aede03e7538de87adbfc499c49b0 Mon Sep 17 00:00:00 2001 From: caoweihan Date: Fri, 1 Apr 2022 16:34:46 +0800 Subject: [PATCH 06/11] split rkd loss to distance-wise and angle-wise losses --- .../rkd_neck_resnet34_resnet18_8xb32_in1k.py | 18 ++--- ...kd_neck_resnet50_vgg11bn_8xb16_cifar100.py | 18 ++--- mmrazor/models/losses/__init__.py | 7 +- .../losses/{rkd.py => relational_kd.py} | 69 ++++++++++++++----- .../test_algorithms/test_algorithm.py | 28 +++++--- 5 files changed, 95 insertions(+), 45 deletions(-) rename mmrazor/models/losses/{rkd.py => relational_kd.py} (64%) diff --git a/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py b/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py index d64b25a5c..2a5829577 100644 --- a/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py +++ b/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py @@ -23,9 +23,7 @@ )) # teacher settings - -# FIXME: replace it with your own path -teacher_ckpt = 'path/to/your/checkpoint.pth' +teacher_ckpt = 'https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_8xb32_in1k_20210831-f257d4e6.pth' # noqa: E501 teacher = dict( type='mmcls.ImageClassifier', @@ -65,11 +63,15 @@ teacher_module='neck.gap', losses=[ dict( - type='RelationalKD', - name='loss_rkd', - loss_weight_d=25.0, - loss_weight_a=50.0, - with_l2_norm=True) + type='Distance_wise_RKD', + name='distance_wise_loss', + loss_weight=25.0, + with_l2_norm=True), + dict( + type='Angle_wise_RKD', + name='angle_wise_loss', + loss_weight=50.0, + with_l2_norm=True), ]) ]), ) diff --git a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py index 9ecc8db40..93b5178e7 100644 --- a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py +++ b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py @@ -18,9 +18,7 @@ )) # teacher settings - -# FIXME: replace it with your own path -teacher_ckpt = 'path/to/your/checkpoint.pth' +teacher_ckpt = 'https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_8xb32_in1k_20210831-f257d4e6.pth' # noqa: E501 teacher = dict( type='mmcls.ImageClassifier', @@ -59,11 +57,15 @@ teacher_module='neck.gap', losses=[ dict( - type='RelationalKD', - name='loss_rkd', - loss_weight_d=25.0, - loss_weight_a=50.0, - with_l2_norm=True) + type='Distance_wise_RKD', + name='distance_wise_loss', + loss_weight=25.0, + with_l2_norm=True), + dict( + type='Angle_wise_RKD', + name='angle_wise_loss', + loss_weight=50.0, + with_l2_norm=True), ]) ]), ) diff --git a/mmrazor/models/losses/__init__.py b/mmrazor/models/losses/__init__.py index 8df360407..112789d63 100644 --- a/mmrazor/models/losses/__init__.py +++ b/mmrazor/models/losses/__init__.py @@ -1,7 +1,10 @@ # Copyright (c) OpenMMLab. All rights reserved. from .cwd import ChannelWiseDivergence from .kl_divergence import KLDivergence -from .rkd import RelationalKD +from .relational_kd import Angle_wise_RKD, Distance_wise_RKD from .weighted_soft_label_distillation import WSLD -__all__ = ['ChannelWiseDivergence', 'KLDivergence', 'RelationalKD', 'WSLD'] +__all__ = [ + 'ChannelWiseDivergence', 'KLDivergence', 'Distance_wise_RKD', + 'Angle_wise_RKD', 'WSLD' +] diff --git a/mmrazor/models/losses/rkd.py b/mmrazor/models/losses/relational_kd.py similarity index 64% rename from mmrazor/models/losses/rkd.py rename to mmrazor/models/losses/relational_kd.py index d20fe9eb2..05307001b 100644 --- a/mmrazor/models/losses/rkd.py +++ b/mmrazor/models/losses/relational_kd.py @@ -47,26 +47,23 @@ def angle(pred): @LOSSES.register_module() -class RelationalKD(nn.Module): - """PyTorch version of `Relational Knowledge Distillation. +class Distance_wise_RKD(nn.Module): + """PyTorch version of distance-wise loss of `Relational Knowledge + Distillation. `_. + Args: - loss_weight_d (float): Weight of distance-wise distillation loss. + loss_weight (float): Weight of distance-wise distillation loss. Defaults to 25.0. - loss_weight_a (float): Weight of angle-wise distillation loss. - Defaults to 50.0. with_l2_norm (bool): Whether to normalize the model predictions before calculating the loss. Defaults to True. """ - def __init__(self, - loss_weight_d=25.0, - loss_weight_a=50.0, - with_l2_norm=True): - super(RelationalKD, self).__init__() - self.loss_weight_d = loss_weight_d - self.loss_weight_a = loss_weight_a + def __init__(self, loss_weight=25.0, with_l2_norm=True): + super(Distance_wise_RKD, self).__init__() + + self.loss_weight = loss_weight self.with_l2_norm = with_l2_norm def distance_loss(self, preds_S, preds_T): @@ -82,6 +79,48 @@ def distance_loss(self, preds_S, preds_T): return F.smooth_l1_loss(d_S, d_T) + def forward(self, preds_S, preds_T): + """Forward computation. + + Args: + preds_S (torch.Tensor): The student model prediction with + shape (N, C, H, W) or shape (N, C). + preds_T (torch.Tensor): The teacher model prediction with + shape (N, C, H, W) or shape (N, C). + Return: + torch.Tensor: The calculated loss value. + """ + preds_S = preds_S.view(preds_S.shape[0], -1) + preds_T = preds_T.view(preds_T.shape[0], -1) + if self.with_l2_norm: + preds_S = F.normalize(preds_S, p=2, dim=1) + preds_T = F.normalize(preds_T, p=2, dim=1) + + loss = self.distance_loss(preds_S, preds_T) * self.loss_weight + + return loss + + +@LOSSES.register_module() +class Angle_wise_RKD(nn.Module): + """PyTorch version of angle-wise loss of `Relational Knowledge + Distillation. + + `_. + + Args: + loss_weight (float): Weight of angle-wise distillation loss. + Defaults to 50.0. + with_l2_norm (bool): Whether to normalize the model predictions before + calculating the loss. Defaults to True. + """ + + def __init__(self, loss_weight=50.0, with_l2_norm=True): + super(Angle_wise_RKD, self).__init__() + + self.loss_weight = loss_weight + self.with_l2_norm = with_l2_norm + def angle_loss(self, preds_S, preds_T): """Calculate the angle-wise distillation loss.""" angle_T = angle(preds_T) @@ -105,10 +144,6 @@ def forward(self, preds_S, preds_T): preds_S = F.normalize(preds_S, p=2, dim=-1) preds_T = F.normalize(preds_T, p=2, dim=-1) - loss = 0. - if self.loss_weight_d > 0: - loss += self.distance_loss(preds_S, preds_T) * self.loss_weight_d - if self.loss_weight_a > 0: - loss += self.angle_loss(preds_S, preds_T) * self.loss_weight_a + loss = self.angle_loss(preds_S, preds_T) * self.loss_weight return loss diff --git a/tests/test_models/test_algorithms/test_algorithm.py b/tests/test_models/test_algorithms/test_algorithm.py index bc2ca6dd8..9d2e6a00b 100644 --- a/tests/test_models/test_algorithms/test_algorithm.py +++ b/tests/test_models/test_algorithms/test_algorithm.py @@ -493,11 +493,15 @@ def test_rkd(): teacher_module='neck.gap', losses=[ dict( - type='RelationalKD', - name='loss_rkd', - loss_weight_d=25.0, - loss_weight_a=50.0, - with_l2_norm=True) + type='Distance_wise_RKD', + name='distance_wise_loss', + loss_weight=25.0, + with_l2_norm=True), + dict( + type='Angle_wise_RKD', + name='angle_wise_loss', + loss_weight=50.0, + with_l2_norm=True), ]) ]), ) @@ -523,11 +527,15 @@ def test_rkd(): teacher_module='neck.gap', losses=[ dict( - type='RelationalKD', - name='loss_rkd', - loss_weight_d=25.0, - loss_weight_a=50.0, - with_l2_norm=False) + type='Distance_wise_RKD', + name='distance_wise_loss', + loss_weight=25.0, + with_l2_norm=False), + dict( + type='Angle_wise_RKD', + name='angle_wise_loss', + loss_weight=50.0, + with_l2_norm=False), ]) ] From c35486982d70d42a0930e8be288ce9c2c35a440c Mon Sep 17 00:00:00 2001 From: caoweihan Date: Fri, 1 Apr 2022 16:54:38 +0800 Subject: [PATCH 07/11] rename rkd losses --- .../distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py | 4 ++-- .../rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py | 4 ++-- mmrazor/models/losses/__init__.py | 6 +++--- mmrazor/models/losses/relational_kd.py | 8 ++++---- tests/test_models/test_algorithms/test_algorithm.py | 8 ++++---- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py b/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py index 2a5829577..f87b1c230 100644 --- a/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py +++ b/configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py @@ -63,12 +63,12 @@ teacher_module='neck.gap', losses=[ dict( - type='Distance_wise_RKD', + type='DistanceWiseRKD', name='distance_wise_loss', loss_weight=25.0, with_l2_norm=True), dict( - type='Angle_wise_RKD', + type='AngleWiseRKD', name='angle_wise_loss', loss_weight=50.0, with_l2_norm=True), diff --git a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py index 93b5178e7..396f04fa2 100644 --- a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py +++ b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py @@ -57,12 +57,12 @@ teacher_module='neck.gap', losses=[ dict( - type='Distance_wise_RKD', + type='DistanceWiseRKD', name='distance_wise_loss', loss_weight=25.0, with_l2_norm=True), dict( - type='Angle_wise_RKD', + type='AngleWiseRKD', name='angle_wise_loss', loss_weight=50.0, with_l2_norm=True), diff --git a/mmrazor/models/losses/__init__.py b/mmrazor/models/losses/__init__.py index 112789d63..3d3e97e52 100644 --- a/mmrazor/models/losses/__init__.py +++ b/mmrazor/models/losses/__init__.py @@ -1,10 +1,10 @@ # Copyright (c) OpenMMLab. All rights reserved. from .cwd import ChannelWiseDivergence from .kl_divergence import KLDivergence -from .relational_kd import Angle_wise_RKD, Distance_wise_RKD +from .relational_kd import AngleWiseRKD, DistanceWiseRKD from .weighted_soft_label_distillation import WSLD __all__ = [ - 'ChannelWiseDivergence', 'KLDivergence', 'Distance_wise_RKD', - 'Angle_wise_RKD', 'WSLD' + 'ChannelWiseDivergence', 'KLDivergence', 'AngleWiseRKD', 'DistanceWiseRKD', + 'WSLD' ] diff --git a/mmrazor/models/losses/relational_kd.py b/mmrazor/models/losses/relational_kd.py index 05307001b..715465809 100644 --- a/mmrazor/models/losses/relational_kd.py +++ b/mmrazor/models/losses/relational_kd.py @@ -47,7 +47,7 @@ def angle(pred): @LOSSES.register_module() -class Distance_wise_RKD(nn.Module): +class DistanceWiseRKD(nn.Module): """PyTorch version of distance-wise loss of `Relational Knowledge Distillation. @@ -61,7 +61,7 @@ class Distance_wise_RKD(nn.Module): """ def __init__(self, loss_weight=25.0, with_l2_norm=True): - super(Distance_wise_RKD, self).__init__() + super(DistanceWiseRKD, self).__init__() self.loss_weight = loss_weight self.with_l2_norm = with_l2_norm @@ -102,7 +102,7 @@ def forward(self, preds_S, preds_T): @LOSSES.register_module() -class Angle_wise_RKD(nn.Module): +class AngleWiseRKD(nn.Module): """PyTorch version of angle-wise loss of `Relational Knowledge Distillation. @@ -116,7 +116,7 @@ class Angle_wise_RKD(nn.Module): """ def __init__(self, loss_weight=50.0, with_l2_norm=True): - super(Angle_wise_RKD, self).__init__() + super(AngleWiseRKD, self).__init__() self.loss_weight = loss_weight self.with_l2_norm = with_l2_norm diff --git a/tests/test_models/test_algorithms/test_algorithm.py b/tests/test_models/test_algorithms/test_algorithm.py index 9d2e6a00b..1869c94e3 100644 --- a/tests/test_models/test_algorithms/test_algorithm.py +++ b/tests/test_models/test_algorithms/test_algorithm.py @@ -493,12 +493,12 @@ def test_rkd(): teacher_module='neck.gap', losses=[ dict( - type='Distance_wise_RKD', + type='DistanceWiseRKD', name='distance_wise_loss', loss_weight=25.0, with_l2_norm=True), dict( - type='Angle_wise_RKD', + type='AngleWiseRKD', name='angle_wise_loss', loss_weight=50.0, with_l2_norm=True), @@ -527,12 +527,12 @@ def test_rkd(): teacher_module='neck.gap', losses=[ dict( - type='Distance_wise_RKD', + type='DistanceWiseRKD', name='distance_wise_loss', loss_weight=25.0, with_l2_norm=False), dict( - type='Angle_wise_RKD', + type='AngleWiseRKD', name='angle_wise_loss', loss_weight=50.0, with_l2_norm=False), From d30de6f726ee7198e19fa70930a673538174c447 Mon Sep 17 00:00:00 2001 From: pppppM Date: Fri, 1 Apr 2022 18:50:00 +0800 Subject: [PATCH 08/11] add rkd metaflie --- configs/distill/rkd/metafile.yaml | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 configs/distill/rkd/metafile.yaml diff --git a/configs/distill/rkd/metafile.yaml b/configs/distill/rkd/metafile.yaml new file mode 100644 index 000000000..7f3c4f19d --- /dev/null +++ b/configs/distill/rkd/metafile.yaml @@ -0,0 +1,48 @@ +Collections: + - Name: RKD + Metadata: + Training Data: + - ImageNet-1k + - CIFAR-100 + Paper: + URL: https://arxiv.org/abs/1904.05068 + Title: Relational Knowledge Distillation + README: configs/distill/rkd/README.md + Code: + URL: https://github.com/open-mmlab/mmrazor/blob/v0.3.0/mmrazor/models/losses/relation_kd.py + Version: v0.3.0 + Converted From: + Code: https://github.com/lenscloth/RKD +Models: + - Name: rkd_neck_resnet34_resnet18_8xb32_in1k + In Collection: RKD + Metadata: + Location: neck + Student: R-18 + Teacher: R-34 + Teacher Checkpoint: https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_8xb32_in1k_20210831-f257d4e6.pth + Results: + - Task: Image Classification + Dataset: ImageNet-1k + Metrics: + Top 1 Accuracy: 70.23 + Top 1 Accuracy:(S): 69.90 + Top 1 Accuracy:(T): 73.62 + Config: configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py + Weights: https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k_acc-70.23_20220401-f25700ac.pth + - Name: rkd_neck_resnet50_vgg11bn_8xb16_cifar100 + In Collection: RKD + Metadata: + Location: neck + Student: R-18 + Teacher: R-50 + Teacher Checkpoint: https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_8xb32_in1k_20210831-f257d4e6.pth + Results: + - Task: Image Classification + Dataset: CIFAR-100 + Metrics: + Top 1 Accuracy: 73.11 + Top 1 Accuracy:(S): 71.26 + Top 1 Accuracy:(T): 79.90 + Config: configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py + Weights: https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100_acc-73.11_20220401-5422b329.pth From ca306fa2c0bdb3b3b1278a07106c71744361884f Mon Sep 17 00:00:00 2001 From: pppppM Date: Fri, 1 Apr 2022 18:50:20 +0800 Subject: [PATCH 09/11] add rkd related links --- configs/distill/rkd/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/distill/rkd/README.md b/configs/distill/rkd/README.md index 8ce6369f8..2a523fd56 100644 --- a/configs/distill/rkd/README.md +++ b/configs/distill/rkd/README.md @@ -26,8 +26,8 @@ teachers' performance, achieving the state of the arts on standard benchmark dat ### Classification |Location|Dataset|Teacher|Student|Acc|Acc(T)|Acc(S)|Config | Download | :--------:|:---------:|:---------:|:---------:|:---------:|:---------:|:---------:|:------:|:---------| -| neck |ImageNet|[resnet34](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet34_8xb32_in1k.py)|[resnet18](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet18_8xb32_in1k.py)| 70.23 | 73.62 | 69.90 |[config](./rkd_neck_resnet34_resnet18_8xb32_in1k.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_b16x8_cifar10_20210528-a8aa36a6.pth) |[model]() | [log]()| -| neck |Cifar100|[resnet50](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet50_8xb16_cifar100.py)|[vgg11bn]()| 72.76 | 79.90 | 71.26 |[config](./rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_b16x8_cifar100_20210528-67b58a1b.pth) |[model]() | [log]()| +| neck |ImageNet|[resnet34](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet34_8xb32_in1k.py)|[resnet18](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet18_8xb32_in1k.py)| 70.23 | 73.62 | 69.90 |[config](./rkd_neck_resnet34_resnet18_8xb32_in1k.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_b16x8_cifar10_20210528-a8aa36a6.pth) |[model](https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k_acc-70.23_20220401-f25700ac.pth) | [log](https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k_20220312_130419.log.json)| +| neck |Cifar100|[resnet50](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet50_8xb16_cifar100.py)|[vgg11bn]()| 73.11 | 79.90 | 71.26 |[config](./rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_b16x8_cifar100_20210528-67b58a1b.pth) |[model](https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100_acc-73.11_20220401-5422b329.pth) | [log](https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100_20220401_124319.log.json)| From c915e09604de60a2e7496c0c1ea14f1ce5a8453f Mon Sep 17 00:00:00 2001 From: pppppM Date: Fri, 1 Apr 2022 18:56:58 +0800 Subject: [PATCH 10/11] rename rkd metafile and add to model index --- configs/distill/rkd/{metafile.yaml => metafile.yml} | 0 model-index.yml | 1 + 2 files changed, 1 insertion(+) rename configs/distill/rkd/{metafile.yaml => metafile.yml} (100%) diff --git a/configs/distill/rkd/metafile.yaml b/configs/distill/rkd/metafile.yml similarity index 100% rename from configs/distill/rkd/metafile.yaml rename to configs/distill/rkd/metafile.yml diff --git a/model-index.yml b/model-index.yml index 8d816a206..d5e316393 100644 --- a/model-index.yml +++ b/model-index.yml @@ -1,6 +1,7 @@ Import: - configs/distill/cwd/metafile.yml - configs/distill/wsld/metafile.yml + - configs/distill/rkd/metafile.yml - configs/nas/darts/metafile.yml - configs/nas/detnas/metafile.yml - configs/nas/spos/metafile.yml From 0132b63155877460d888636bb122df7a00e7ceb4 Mon Sep 17 00:00:00 2001 From: caoweihan Date: Sat, 2 Apr 2022 14:14:09 +0800 Subject: [PATCH 11/11] delete cifar100 --- configs/distill/rkd/README.md | 1 - configs/distill/rkd/metafile.yml | 17 ----- ...kd_neck_resnet50_vgg11bn_8xb16_cifar100.py | 76 ------------------- 3 files changed, 94 deletions(-) delete mode 100644 configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py diff --git a/configs/distill/rkd/README.md b/configs/distill/rkd/README.md index 2a523fd56..ef9095634 100644 --- a/configs/distill/rkd/README.md +++ b/configs/distill/rkd/README.md @@ -27,7 +27,6 @@ teachers' performance, achieving the state of the arts on standard benchmark dat |Location|Dataset|Teacher|Student|Acc|Acc(T)|Acc(S)|Config | Download | :--------:|:---------:|:---------:|:---------:|:---------:|:---------:|:---------:|:------:|:---------| | neck |ImageNet|[resnet34](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet34_8xb32_in1k.py)|[resnet18](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet18_8xb32_in1k.py)| 70.23 | 73.62 | 69.90 |[config](./rkd_neck_resnet34_resnet18_8xb32_in1k.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_b16x8_cifar10_20210528-a8aa36a6.pth) |[model](https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k_acc-70.23_20220401-f25700ac.pth) | [log](https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k_20220312_130419.log.json)| -| neck |Cifar100|[resnet50](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet50_8xb16_cifar100.py)|[vgg11bn]()| 73.11 | 79.90 | 71.26 |[config](./rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_b16x8_cifar100_20210528-67b58a1b.pth) |[model](https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100_acc-73.11_20220401-5422b329.pth) | [log](https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100_20220401_124319.log.json)| diff --git a/configs/distill/rkd/metafile.yml b/configs/distill/rkd/metafile.yml index 7f3c4f19d..3d5380eaa 100644 --- a/configs/distill/rkd/metafile.yml +++ b/configs/distill/rkd/metafile.yml @@ -3,7 +3,6 @@ Collections: Metadata: Training Data: - ImageNet-1k - - CIFAR-100 Paper: URL: https://arxiv.org/abs/1904.05068 Title: Relational Knowledge Distillation @@ -30,19 +29,3 @@ Models: Top 1 Accuracy:(T): 73.62 Config: configs/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k.py Weights: https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet34_resnet18_8xb32_in1k_acc-70.23_20220401-f25700ac.pth - - Name: rkd_neck_resnet50_vgg11bn_8xb16_cifar100 - In Collection: RKD - Metadata: - Location: neck - Student: R-18 - Teacher: R-50 - Teacher Checkpoint: https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_8xb32_in1k_20210831-f257d4e6.pth - Results: - - Task: Image Classification - Dataset: CIFAR-100 - Metrics: - Top 1 Accuracy: 73.11 - Top 1 Accuracy:(S): 71.26 - Top 1 Accuracy:(T): 79.90 - Config: configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py - Weights: https://download.openmmlab.com/mmrazor/v0.3/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100_acc-73.11_20220401-5422b329.pth diff --git a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py b/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py deleted file mode 100644 index 396f04fa2..000000000 --- a/configs/distill/rkd/rkd_neck_resnet50_vgg11bn_8xb16_cifar100.py +++ /dev/null @@ -1,76 +0,0 @@ -_base_ = [ - '../../_base_/datasets/mmcls/cifar100_bs16.py', - '../../_base_/schedules/mmcls/cifar10_bs128.py', - '../../_base_/mmcls_runtime.py' -] - -# model settings -student = dict( - type='mmcls.ImageClassifier', - backbone=dict(type='VGG', depth=11, norm_cfg=dict(type='BN')), - neck=dict(type='GlobalAveragePooling'), - head=dict( - type='LinearClsHead', - num_classes=100, - in_channels=512, - loss=dict(type='CrossEntropyLoss', loss_weight=1.0), - topk=(1, 5), - )) - -# teacher settings -teacher_ckpt = 'https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_8xb32_in1k_20210831-f257d4e6.pth' # noqa: E501 - -teacher = dict( - type='mmcls.ImageClassifier', - init_cfg=dict(type='Pretrained', checkpoint=teacher_ckpt), - backbone=dict( - type='ResNet_CIFAR', - depth=50, - num_stages=4, - out_indices=(3, ), - style='pytorch'), - neck=dict(type='GlobalAveragePooling'), - head=dict( - type='LinearClsHead', - num_classes=100, - in_channels=2048, - loss=dict(type='CrossEntropyLoss', loss_weight=1.0), - )) - -# algorithm setting -algorithm = dict( - type='GeneralDistill', - architecture=dict( - type='MMClsArchitecture', - model=student, - ), - with_student_loss=True, - with_teacher_loss=False, - distiller=dict( - type='SingleTeacherDistiller', - teacher=teacher, - teacher_trainable=False, - teacher_norm_eval=True, - components=[ - dict( - student_module='neck.gap', - teacher_module='neck.gap', - losses=[ - dict( - type='DistanceWiseRKD', - name='distance_wise_loss', - loss_weight=25.0, - with_l2_norm=True), - dict( - type='AngleWiseRKD', - name='angle_wise_loss', - loss_weight=50.0, - with_l2_norm=True), - ]) - ]), -) - -find_unused_parameters = True - -optimizer = dict(type='SGD', lr=0.1, momentum=0.9, weight_decay=0.0005) -lr_config = dict(policy='step', step=[60, 120, 160], gamma=0.2)