diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8ac46c21..2e517e380 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,72 +3,76 @@ name: build -on: [push, pull_request] +on: + push: + paths-ignore: + - "README.md" + - "README_zh-CN.md" + - "model-index.yml" + - "configs/**" + - "docs/**" + - ".dev_scripts/**" -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install pre-commit hook - run: | - pip install pre-commit - pre-commit install - - name: Linting - run: pre-commit run --all-files + pull_request: + paths-ignore: + - "README.md" + - "README_zh-CN.md" + - "model-index.yml" + - "configs/**" + - "docs/**" + - ".dev_scripts/**" - build: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test_linux: runs-on: ubuntu-latest - env: - UBUNTU_VERSION: ubuntu1804 strategy: matrix: - python-version: [3.7] - torch: [1.5.0, 1.6.0, 1.7.0, 1.8.0, 1.9.0] + torch: + [1.5.1+cpu, 1.6.0+cpu, 1.7.1+cpu, 1.8.0+cpu, 1.9.0+cpu, 1.10.1+cpu] include: - - torch: 1.5.0 - torchvision: 0.6.0 - - torch: 1.6.0 - torchvision: 0.7.0 - - torch: 1.7.0 - torchvision: 0.8.1 - - torch: 1.8.0 - torchvision: 0.9.0 - - torch: 1.8.0 - torchvision: 0.9.0 + - torch: 1.5.1+cpu + torchvision: 0.6.1+cpu + mmcv_link: cpu/torch1.5 + python-version: 3.7 + - torch: 1.6.0+cpu + torchvision: 0.7.0+cpu + mmcv_link: cpu/torch1.6 + python-version: 3.7 + - torch: 1.7.1+cpu + torchvision: 0.8.2+cpu + mmcv_link: cpu/torch1.7 + python-version: 3.7 + - torch: 1.8.0+cpu + torchvision: 0.9.0+cpu + mmcv_link: cpu/torch1.8 python-version: 3.8 - - torch: 1.8.0 - torchvision: 0.9.0 - python-version: 3.9 - - torch: 1.9.0 - torchvision: 0.10.0 - - torch: 1.9.0 - torchvision: 0.10.0 + - torch: 1.9.0+cpu + torchvision: 0.10.0+cpu + mmcv_link: cpu/torch1.9 python-version: 3.8 - - torch: 1.9.0 - torchvision: 0.10.0 + - torch: 1.10.1+cpu + torchvision: 0.11.2+cpu + mmcv_link: cpu/torch1.10 python-version: 3.9 - steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install Pillow - run: pip install Pillow==6.2.2 - if: ${{matrix.torchvision < 0.5}} + - name: Upgrade pip + run: pip install pip --upgrade - name: Install PyTorch - run: pip install --use-deprecated=legacy-resolver torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html + run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html - name: Install MMCV run: | - pip install --use-deprecated=legacy-resolver mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.torch}}/index.html + pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/${{matrix.mmcv_link}}/index.html python -c 'import mmcv; print(mmcv.__version__)' - - name: Install mmcls dependencies + - name: Install unittest dependencies run: | pip install -r requirements.txt - name: Install timm @@ -85,8 +89,55 @@ jobs: coverage report -m --omit="mmrazor/apis/*" # Only upload coverage report for python3.7 && pytorch1.5 - name: Upload coverage to Codecov - if: ${{matrix.torch == '1.5.0' && matrix.python-version == '3.7'}} - uses: codecov/codecov-action@v1.0.10 + if: ${{matrix.torch == '1.5.1+cpu' && matrix.python-version == '3.7'}} + uses: codecov/codecov-action@v2 + with: + file: ./coverage.xml + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false + + test_windows: + runs-on: windows-latest + strategy: + matrix: + torch: [1.8.2+cpu] + torchvision: [0.9.2+cpu] + mmcv_link: [cpu/torch1.8] + python-version: [3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: pip install pip --upgrade --user + - name: Install PyTorch + run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html + - name: Install OpenCV + run: | + pip install opencv-python>=3 + - name: Install MMCV + run: | + pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.8/index.html --only-binary mmcv-full + - name: Install unittest dependencies + run: | + pip install -r requirements.txt + - name: Install timm + run: | + pip install timm + - name: Build and install + run: | + pip install -e . -U + - name: Run unittests and generate coverage report + run: | + coverage run --branch --source mmrazor -m pytest tests/ + coverage xml + coverage report -m --omit="mmrazor/apis/*" + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 with: file: ./coverage.xml flags: unittests diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..36422d008 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,27 @@ +name: lint + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install pre-commit hook + run: | + pip install pre-commit + pre-commit install + - name: Linting + run: pre-commit run --all-files + - name: Check docstring coverage + run: | + pip install interrogate + interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-regex "__repr__" --fail-under 80 mmrazor diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19e9f8d48..51f54d849 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,12 +4,8 @@ repos: rev: 3.8.3 hooks: - id: flake8 - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-yapf @@ -29,8 +25,8 @@ repos: args: ["--remove"] - id: mixed-line-ending args: ["--fix=lf"] - - repo: https://github.com/jumanjihouse/pre-commit-hooks - rev: 2.1.4 + - repo: https://github.com/markdownlint/markdownlint + rev: v0.11.0 hooks: - id: markdownlint args: ["-r", "~MD002,~MD013,~MD029,~MD033,~MD034", @@ -44,11 +40,9 @@ repos: hooks: - id: docformatter args: ["--in-place", "--wrap-descriptions", "79"] - # - repo: local - # hooks: - # - id: clang-format - # name: clang-format - # description: Format files with ClangFormat - # entry: clang-format -style=google -i - # language: system - # files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|cuh|proto)$ + - repo: https://github.com/open-mmlab/pre-commit-hooks + rev: v0.2.0 + hooks: + - id: check-algo-readme + - id: check-copyright + args: [ "mmrazor", "tests", "tools"] diff --git a/README.md b/README.md index c6c622d64..841bd7419 100644 --- a/README.md +++ b/README.md @@ -117,20 +117,21 @@ We wish that the toolbox and benchmark could serve the growing research communit ## Projects in OpenMMLab - [MMCV](https://github.com/open-mmlab/mmcv): OpenMMLab foundational library for computer vision. -- [MIM](https://github.com/open-mmlab/mim): MIM Installs OpenMMLab Packages. +- [MIM](https://github.com/open-mmlab/mim): MIM installs OpenMMLab packages. - [MMClassification](https://github.com/open-mmlab/mmclassification): OpenMMLab image classification toolbox and benchmark. - [MMDetection](https://github.com/open-mmlab/mmdetection): OpenMMLab detection toolbox and benchmark. -- [MMDetection3D](https://github.com/open-mmlab/mmdetection3d): OpenMMLab next-generation platform for general 3D object detection. +- [MMDetection3D](https://github.com/open-mmlab/mmdetection3d): OpenMMLab's next-generation platform for general 3D object detection. +- [MMRotate](https://github.com/open-mmlab/mmrotate): OpenMMLab rotated object detection toolbox and benchmark. - [MMSegmentation](https://github.com/open-mmlab/mmsegmentation): OpenMMLab semantic segmentation toolbox and benchmark. -- [MMAction2](https://github.com/open-mmlab/mmaction2): OpenMMLab next-generation action understanding toolbox and benchmark. -- [MMTracking](https://github.com/open-mmlab/mmtracking): OpenMMLab video perception toolbox and benchmark. +- [MMOCR](https://github.com/open-mmlab/mmocr): OpenMMLab text detection, recognition, and understanding toolbox. - [MMPose](https://github.com/open-mmlab/mmpose): OpenMMLab pose estimation toolbox and benchmark. +- [MMHuman3D](https://github.com/open-mmlab/mmhuman3d): OpenMMLab 3D human parametric model toolbox and benchmark. +- [MMSelfSup](https://github.com/open-mmlab/mmselfsup): OpenMMLab self-supervised learning toolbox and benchmark. +- [MMRazor](https://github.com/open-mmlab/mmrazor): OpenMMLab model compression toolbox and benchmark. +- [MMFewShot](https://github.com/open-mmlab/mmfewshot): OpenMMLab fewshot learning toolbox and benchmark. +- [MMAction2](https://github.com/open-mmlab/mmaction2): OpenMMLab's next-generation action understanding toolbox and benchmark. +- [MMTracking](https://github.com/open-mmlab/mmtracking): OpenMMLab video perception toolbox and benchmark. +- [MMFlow](https://github.com/open-mmlab/mmflow): OpenMMLab optical flow toolbox and benchmark. - [MMEditing](https://github.com/open-mmlab/mmediting): OpenMMLab image and video editing toolbox. -- [MMOCR](https://github.com/open-mmlab/mmocr): OpenMMLab toolbox for text detection, recognition and understanding. -- [MMGeneration](https://github.com/open-mmlab/mmgeneration): OpenMMlab toolkit for generative models. -- [MMFlow](https://github.com/open-mmlab/mmflow) OpenMMLab optical flow toolbox and benchmark. -- [MMFewShot](https://github.com/open-mmlab/mmfewshot): OpenMMLab FewShot Learning Toolbox and Benchmark. -- [MMHuman3D](https://github.com/open-mmlab/mmhuman3d): OpenMMLab 3D Human Parametric Model Toolbox and Benchmark. -- [MMSelfSup](https://github.com/open-mmlab/mmselfsup): OpenMMLab self-supervised learning Toolbox and Benchmark. -- [MMRazor](https://github.com/open-mmlab/mmrazor): OpenMMLab Model Compression Toolbox and Benchmark. -- [MMDeploy](https://github.com/open-mmlab/mmdeploy): OpenMMLab Model Deployment Framework. +- [MMGeneration](https://github.com/open-mmlab/mmgeneration): OpenMMLab image and video generative models toolbox. +- [MMDeploy](https://github.com/open-mmlab/mmdeploy): OpenMMLab model deployment framework. diff --git a/README_zh-CN.md b/README_zh-CN.md index 08a4e8a2c..8dfefd2aa 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -146,20 +146,20 @@ MMRazor 是一款由来自不同高校和企业的研发人员共同参与贡献 - [MMClassification](https://github.com/open-mmlab/mmclassification): OpenMMLab 图像分类工具箱 - [MMDetection](https://github.com/open-mmlab/mmdetection): OpenMMLab 目标检测工具箱 - [MMDetection3D](https://github.com/open-mmlab/mmdetection3d): OpenMMLab 新一代通用 3D 目标检测平台 +- [MMRotate](https://github.com/open-mmlab/mmrotate): OpenMMLab 旋转框检测工具箱与测试基准 - [MMSegmentation](https://github.com/open-mmlab/mmsegmentation): OpenMMLab 语义分割工具箱 -- [MMAction2](https://github.com/open-mmlab/mmaction2): OpenMMLab 新一代视频理解工具箱 -- [MMTracking](https://github.com/open-mmlab/mmtracking): OpenMMLab 一体化视频目标感知平台 +- [MMOCR](https://github.com/open-mmlab/mmocr): OpenMMLab 全流程文字检测识别理解工具箱 - [MMPose](https://github.com/open-mmlab/mmpose): OpenMMLab 姿态估计工具箱 -- [MMEditing](https://github.com/open-mmlab/mmediting): OpenMMLab 图像视频编辑工具箱 -- [MMOCR](https://github.com/open-mmlab/mmocr): OpenMMLab 全流程文字检测识别理解工具包 -- [MMGeneration](https://github.com/open-mmlab/mmgeneration): OpenMMLab 图片视频生成模型工具箱 -- [MMFlow](https://github.com/open-mmlab/mmflow): OpenMMLab 光流估计工具箱与测试基准 -- [MMFewShot](https://github.com/open-mmlab/mmfewshot): OpenMMLab 少样本学习工具箱与测试基准 - [MMHuman3D](https://github.com/open-mmlab/mmhuman3d): OpenMMLab 人体参数化模型工具箱与测试基准 - [MMSelfSup](https://github.com/open-mmlab/mmselfsup): OpenMMLab 自监督学习工具箱与测试基准 - [MMRazor](https://github.com/open-mmlab/mmrazor): OpenMMLab 模型压缩工具箱与测试基准 +- [MMFewShot](https://github.com/open-mmlab/mmfewshot): OpenMMLab 少样本学习工具箱与测试基准 +- [MMAction2](https://github.com/open-mmlab/mmaction2): OpenMMLab 新一代视频理解工具箱 +- [MMTracking](https://github.com/open-mmlab/mmtracking): OpenMMLab 一体化视频目标感知平台 +- [MMFlow](https://github.com/open-mmlab/mmflow): OpenMMLab 光流估计工具箱与测试基准 +- [MMEditing](https://github.com/open-mmlab/mmediting): OpenMMLab 图像视频编辑工具箱 +- [MMGeneration](https://github.com/open-mmlab/mmgeneration): OpenMMLab 图片视频生成模型工具箱 - [MMDeploy](https://github.com/open-mmlab/mmdeploy): OpenMMLab 模型部署框架 - ## 欢迎加入 OpenMMLab 社区 扫描下方的二维码可关注 OpenMMLab 团队的 [知乎官方账号](https://www.zhihu.com/people/openmmlab)，加入 OpenMMLab 团队的 [官方交流 QQ 群](https://jq.qq.com/?_wv=1027&k=aCvMxdr3) diff --git a/configs/distill/cwd/README.md b/configs/distill/cwd/README.md index 5e540cc88..9033221ce 100644 --- a/configs/distill/cwd/README.md +++ b/configs/distill/cwd/README.md @@ -1,22 +1,15 @@ -# CHANNEL-WISE KNOWLEDGE DISTILLATION FOR DENSE PREDICTION +# CWD +> [Channel-wise Knowledge Distillation for Dense Prediction](https://arxiv.org/abs/2011.13256) + ## Abstract Knowledge distillation (KD) has been proven to be a simple and effective tool for training compact models. Almost all KD variants for dense prediction tasks align the student and teacher networks' feature maps in the spatial domain, typically by minimizing point-wise and/or pair-wise discrepancy. Observing that in semantic segmentation, some layers' feature activations of each channel tend to encode saliency of scene categories (analogue to class activation mapping), we propose to align features channel-wise between the student and teacher networks. To this end, we first transform the feature map of each channel into a probability map using softmax normalization, and then minimize the Kullback-Leibler (KL) divergence of the corresponding channels of the two networks. By doing so, our method focuses on mimicking the soft distributions of channels between networks. In particular, the KL divergence enables learning to pay more attention to the most salient regions of the channel-wise maps, presumably corresponding to the most useful signals for semantic segmentation. Experiments demonstrate that our channel-wise distillation outperforms almost all existing spatial distillation methods for semantic segmentation considerably, and requires less computational cost during training. We consistently achieve superior performance on three benchmarks with various network structures. + ![pipeline](/docs/en/imgs/model_zoo/cwd/pipeline.png) -## Citation -```latex -@inproceedings{shu2021channel, - title={Channel-Wise Knowledge Distillation for Dense Prediction}, - author={Shu, Changyong and Liu, Yifan and Gao, Jianfei and Yan, Zheng and Shen, Chunhua}, - booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, - pages={5311--5320}, - year={2021} -} -``` ## Results and models ### Segmentation @@ -28,3 +21,16 @@ Knowledge distillation (KD) has been proven to be a simple and effective tool fo |Location|Dataset|Teacher|Student|mAP|mAP(T)|mAP(S)|Config | Download | :--------:|:---------:|:---------:|:---------:|:---------:|:---------:|:---------:|:------:|:---------| | cls head |COCO|[gfl_r101_2x](https://github.com/open-mmlab/mmdetection/tree/master/configs/gfl/gfl_r101_fpn_mstrain_2x_coco.py)|[gfl_r50_1x](https://github.com/open-mmlab/mmdetection/tree/master/configs/gfl/gfl_r50_fpn_1x_coco.py)| 41.9 | 44.7 | 40.2 |[config]()|[teacher](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126-dd12f847.pth) |[model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/distill/cwd/cwd_cls_head_gfl_r101_fpn_gfl_r50_fpn_1x_coco/cwd_cls_head_gfl_r101_fpn_gfl_r50_fpn_1x_coco_20211222-655dff39.pth?versionId=CAEQHxiBgMD7.uuI7xciIDY1MDRjYzlkN2ExOTRiY2NhNmU4NGJlMmExNjA2YzMy) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/distill/cwd/cwd_cls_head_gfl_r101_fpn_gfl_r50_fpn_1x_coco/cwd_cls_head_gfl_r101_fpn_gfl_r50_fpn_1x_coco_20211212_205444.log.json?versionId=CAEQHxiBgID.o_WI7xciIDgyZjRjYTU4Y2ZjNjRjOGU5MTBlMTQ3ZjEyMTE4OTJl)| + + +## Citation + +```latex +@inproceedings{shu2021channel, + title={Channel-Wise Knowledge Distillation for Dense Prediction}, + author={Shu, Changyong and Liu, Yifan and Gao, Jianfei and Yan, Zheng and Shen, Chunhua}, + booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, + pages={5311--5320}, + year={2021} +} +``` diff --git a/configs/distill/cwd/metafile.yml b/configs/distill/cwd/metafile.yml new file mode 100644 index 000000000..4b1e155ae --- /dev/null +++ b/configs/distill/cwd/metafile.yml @@ -0,0 +1,50 @@ +Collections: + - Name: CWD + Metadata: + Training Data: + - Cityscapes + - COCO + Paper: + URL: https://arxiv.org/abs/2011.13256 + Title: Channel-wise Knowledge Distillation for Dense Prediction + README: configs/distill/cwd/README.md + Code: + URL: https://github.com/open-mmlab/mmrazor/blob/v0.1.0/mmrazor/models/losses/cwd.py#L10 + Version: v0.1.0 + Converted From: + Code: + - https://github.com/pppppM/mmsegmentation-distiller + - https://github.com/pppppM/mmdetection-distiller +Models: + - Name: cwd_cls_head_pspnet_r101_d8_pspnet_r18_d8_512x1024_cityscapes_80k + In Collection: CWD + Metadata: + Location: cls head + Student: pspnet-r18-d8 + Teacher: pspnet-r101-d8 + Teacher Checkpoint: https://download.openmmlab.com/mmsegmentation/v0.5/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes/pspnet_r101-d8_512x1024_80k_cityscapes_20200606_112211-e1e1100f.pth + Results: + - Task: Semantic Segmentation + Dataset: Cityscapes + Metrics: + mIoU: 75.54 + mIoU(S): 74.87 + mIoU(T): 79.76 + Config: configs/distill/cwd/cwd_cls_head_pspnet_r101_d8_pspnet_r18_d8_512x1024_cityscapes_80k.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/distill/cwd/cwd_cls_head_pspnet_r101_d8_pspnet_r18_d8_512x1024_cityscapes_80k/cwd_cls_head_pspnet_r101_d8_pspnet_r18_d8_512x1024_cityscapes_80k_mIoU-75.54_20211222-3a26ee1c.pth + - Name: cwd_cls_head_gfl_r101_fpn_gfl_r50_fpn_1x_coco + In Collection: CWD + Metadata: + Location: cls head + Student: gfl-r50-fpn + Teacher: gfl-r101-fpn + Teacher Checkpoint: https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126-dd12f847.pth + Results: + - Task: Object Detection + Dataset: COCO + Metrics: + box AP: 41.9 + box AP(S): 40.2 + box AP(T): 44.7 + Config: configs/distill/cwd/cwd_cls_head_gfl_r101_fpn_gfl_r50_fpn_1x_coco.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/distill/cwd/cwd_cls_head_gfl_r101_fpn_gfl_r50_fpn_1x_coco/cwd_cls_head_gfl_r101_fpn_gfl_r50_fpn_1x_coco_20211222-655dff39.pth diff --git a/configs/distill/wsld/README.md b/configs/distill/wsld/README.md index c5ffc47ad..10f259b3b 100644 --- a/configs/distill/wsld/README.md +++ b/configs/distill/wsld/README.md @@ -1,5 +1,11 @@ -# RETHINKING SOFT LABELS FOR KNOWLEDGE DISTILLATION: A BIAS-VARIANCE TRADEOFF PERSPECTIVE +# WSLD + + +> [Rethinking Soft Labels for Knowledge Distillation: A Bias-Variance Tradeoff Perspective](https://arxiv.org/abs/2102.00650) + + +## Abstract Knowledge distillation is an effective approach to leverage a well-trained network or an ensemble of them, named as the teacher, to guide the training of a student network. The outputs from the teacher network are used as soft labels for supervising the training of a new network. Recent studies (Muller et al., 2019; Yuan ¨ @@ -16,6 +22,15 @@ weighted soft labels to help the network adaptively handle the sample-wise biasv effectiveness of our method. ![pipeline](/docs/en/imgs/model_zoo/wsld/pipeline.png) + +## Results and models +### Classification +|Location|Dataset|Teacher|Student|Acc|Acc(T)|Acc(S)|Config | Download | +:--------:|:---------:|:---------:|:---------:|:---------:|:---------:|:---------:|:------:|:---------| +| cls head |ImageNet|[resnet34](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet34_8xb32_in1k.py)|[resnet18](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet18_8xb32_in1k.py)| 71.54 | 73.62 | 69.90 |[config](./wsld_cls_head_resnet34_resnet18_8xb32_in1k.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_8xb32_in1k_20210831-ea4938fc.pth) |[model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/distill/wsld/wsld_cls_head_resnet34_resnet18_8xb32_in1k/wsld_cls_head_resnet34_resnet18_8xb32_in1k_acc-71.54_20211222-91f28cf6.pth?versionId=CAEQHxiBgMC6memK7xciIGMzMDFlYTA4YzhlYTRiMTNiZWU0YTVhY2I5NjVkMjY2) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/distill/wsld/wsld_cls_head_resnet34_resnet18_8xb32_in1k/wsld_cls_head_resnet34_resnet18_8xb32_in1k_20211221_181516.log.json?versionId=CAEQHxiBgIDLmemK7xciIGNkM2FiN2Y4N2E5YjRhNDE4NDVlNmExNDczZDIxN2E5)| + + + ## Citation ```latex @inproceedings{zhou2021wsl, @@ -25,9 +40,3 @@ effectiveness of our method. year={2021} } ``` - -## Results and models -### Classification -|Location|Dataset|Teacher|Student|Acc|Acc(T)|Acc(S)|Config | Download | -:--------:|:---------:|:---------:|:---------:|:---------:|:---------:|:---------:|:------:|:---------| -| cls head |ImageNet|[resnet34](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet34_8xb32_in1k.py)|[resnet18](https://github.com/open-mmlab/mmclassification/blob/master/configs/resnet/resnet18_8xb32_in1k.py)| 71.54 | 73.62 | 69.90 |[config](./wsld_cls_head_resnet34_resnet18_8xb32_in1k.py)|[teacher](https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_8xb32_in1k_20210831-ea4938fc.pth) |[model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/distill/wsld/wsld_cls_head_resnet34_resnet18_8xb32_in1k/wsld_cls_head_resnet34_resnet18_8xb32_in1k_acc-71.54_20211222-91f28cf6.pth?versionId=CAEQHxiBgMC6memK7xciIGMzMDFlYTA4YzhlYTRiMTNiZWU0YTVhY2I5NjVkMjY2) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/distill/wsld/wsld_cls_head_resnet34_resnet18_8xb32_in1k/wsld_cls_head_resnet34_resnet18_8xb32_in1k_20211221_181516.log.json?versionId=CAEQHxiBgIDLmemK7xciIGNkM2FiN2Y4N2E5YjRhNDE4NDVlNmExNDczZDIxN2E5)| diff --git a/configs/distill/wsld/metafile.yml b/configs/distill/wsld/metafile.yml new file mode 100644 index 000000000..f807e3237 --- /dev/null +++ b/configs/distill/wsld/metafile.yml @@ -0,0 +1,31 @@ +Collections: + - Name: WSLD + Metadata: + Training Data: + - ImageNet-1k + Paper: + URL: https://arxiv.org/abs/2102.00650 + Title: Rethinking Soft Labels for Knowledge Distillation:A Bias-Variance Tradeoff Perspective + README: configs/distill/wsld/README.md + Code: + URL: https://github.com/open-mmlab/mmrazor/blob/v0.1.0/mmrazor/models/losses/weighted_soft_label_distillation.py + Version: v0.1.0 + Converted From: + Code: https://github.com/bellymonster/Weighted-Soft-Label-Distillation +Models: + - Name: wsld_cls_head_resnet34_resnet18_8xb32_in1k + In Collection: WSLD + Metadata: + Location: cls head + Student: R-18 + Teacher: R-34 + Teacher Checkpoint: https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_8xb32_in1k_20210831-ea4938fc.pth + Results: + - Task: Image Classification + Dataset: ImageNet-1k + Metrics: + Top 1 Accuracy: 71.54 + Top 1 Accuracy:(S): 69.90 + Top 1 Accuracy:(T): 73.62 + Config: configs/distill/wsld/wsld_cls_head_resnet34_resnet18_8xb32_in1k.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/distill/wsld/wsld_cls_head_resnet34_resnet18_8xb32_in1k/wsld_cls_head_resnet34_resnet18_8xb32_in1k_acc-71.54_20211222-91f28cf6.pth diff --git a/configs/nas/darts/README.md b/configs/nas/darts/README.md index eac64dbef..3eb4d38d3 100644 --- a/configs/nas/darts/README.md +++ b/configs/nas/darts/README.md @@ -1,21 +1,14 @@ -# DARTS: DIFFERENTIABLE ARCHITECTURE SEARCH +# DARTS +> [DARTS: Differentiable Architecture Search](https://arxiv.org/abs/1806.09055) + ## Abstract This paper addresses the scalability challenge of architecture search by formulating the task in a differentiable manner. Unlike conventional approaches of applying evolution or reinforcement learning over a discrete and non-differentiable search space, our method is based on the continuous relaxation of the architecture representation, allowing efficient search of the architecture using gradient descent. Extensive experiments on CIFAR-10, ImageNet, Penn Treebank and WikiText-2 show that our algorithm excels in discovering high-performance convolutional architectures for image classification and recurrent architectures for language modeling, while being orders of magnitude faster than state-of-the-art non-differentiable techniques. Our implementation has been made publicly available to facilitate further research on efficient architecture search algorithms. ![pipeline](/docs/en/imgs/model_zoo/darts/pipeline.png) -## Citation -```latex -@inproceedings{liu2018darts, - title={DARTS: Differentiable Architecture Search}, - author={Liu, Hanxiao and Simonyan, Karen and Yang, Yiming}, - booktitle={International Conference on Learning Representations}, - year={2018} -} -``` ## Results and models @@ -31,3 +24,15 @@ Dataset|Params(M)|Flops(G)|Top-1 Acc|Top-5 Acc|Subnet|Config|Download|Remarks| |:---------:|:---------:|:---------:|:---------:|:---------:|:---------:|:------:|:------:|:------:| |Cifar10|3.42 | 0.48 | 97.32 |99.94|[mutable](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/darts/darts_subnetnet_1xb96_cifar10/darts_subnetnet_1xb96_cifar10_acc-97.32_20211222-e5727921_mutable_cfg.yaml?versionId=CAEQHxiBgMDn0ICL7xciIDAwNzUzZTU3ZjE4OTQ0MDg5YmZiMmYzYzExZTQ3YTRm)|[config](./darts_subnetnet_1xb96_cifar10.py)| [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/darts/darts_subnetnet_1xb96_cifar10/darts_subnetnet_1xb96_cifar10_acc-97.32_20211222-e5727921.pth?versionId=CAEQHxiBgID20ICL7xciIDllOWZmNTliMzkwNzQ5YzdhODk2MzY1MWEyOTQ1Yjlk) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/darts/darts_subnetnet_1xb96_cifar10/darts_subnetnet_1xb96_cifar10_20211222-e5727921.log.json?versionId=CAEQHxiBgMDz0ICL7xciIGRhMjk0NDU0OTVhZjQwMDg4N2ZkMDAzZDM1ZWU4N2Ri)|MMRazor searched |Cifar10|3.83 | 0.55 | 97.27 |99.98|[mutable](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/darts/darts_subnetnet_1xb96_cifar10/darts_subnetnet_1xb96_cifar10_acc-97.27_20211222-17e42600_mutable_cfg.yaml?versionId=CAEQHxiBgICrnpmL7xciIGFmYzUxYjdmYWM1YzQ3N2I5NGU1MDE2ZjIxYmJhY2E0)|[config](./darts_subnetnet_1xb96_cifar10.py)| [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/darts/darts_subnetnet_1xb96_cifar10/darts_subnetnet_1xb96_cifar10_acc-97.27_20211222-17e42600.pth?versionId=CAEQHxiBgIDQnpmL7xciIGQzOTRkMTViMDgzNzQ2MWI5MmUyNzIxZDk4OTUzZDgz) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/darts/darts_subnetnet_1xb96_cifar10/darts_subnetnet_1xb96_cifar10_20211222-17e42600.log.json?versionId=CAEQHxiBgMDPnpmL7xciIDViYTVlYTIyYmQ2OTQ1ZDZhNTNhMjVkODA2NDRlMTI1)|official + + +## Citation + +```latex +@inproceedings{liu2018darts, + title={DARTS: Differentiable Architecture Search}, + author={Liu, Hanxiao and Simonyan, Karen and Yang, Yiming}, + booktitle={International Conference on Learning Representations}, + year={2018} +} +``` diff --git a/configs/nas/darts/metafile.yml b/configs/nas/darts/metafile.yml new file mode 100644 index 000000000..2692f3782 --- /dev/null +++ b/configs/nas/darts/metafile.yml @@ -0,0 +1,28 @@ +Collections: + - Name: Darts + Metadata: + Training Data: + - CIFAR-10 + Paper: + URL: https://arxiv.org/abs/1806.09055 + Title: DARTS:Differentiable Architecture Search + README: configs/nas/darts/README.md + Code: + URL: https://github.com/open-mmlab/mmrazor/blob/v0.1.0/mmrazor/models/algorithms/darts.py + Version: v0.1.0 + Converted From: + Code: https://github.com/quark0/darts +Models: + - Name: darts_subnetnet_1xb96_cifar10 + In Collection: Darts + Metadata: + Params(M): 3.42 + Mutable: https://download.openmmlab.com/mmrazor/v0.1/nas/darts/darts_subnetnet_1xb96_cifar10/darts_subnetnet_1xb96_cifar10_acc-97.32_20211222-e5727921_mutable_cfg.yaml + Results: + - Task: Image Classification + Dataset: CIFAR-10 + Metrics: + Top 1 Accuracy: 97.32 + Top 5 Accuracy: 99.94 + Config: configs/nas/darts/darts_subnetnet_1xb96_cifar10.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/nas/darts/darts_subnetnet_1xb96_cifar10/darts_subnetnet_1xb96_cifar10_acc-97.32_20211222-e5727921.pth diff --git a/configs/nas/detnas/README.md b/configs/nas/detnas/README.md index 7c09dc689..60fb57a91 100644 --- a/configs/nas/detnas/README.md +++ b/configs/nas/detnas/README.md @@ -1,35 +1,15 @@ -# DETNAS: BACKBONE SEARCH FOR OBJECT DETECTION +# DetNAS +> [DetNAS: Backbone Search for Object Detection](https://arxiv.org/abs/1903.10979) + ## Abstract Object detectors are usually equipped with backbone networks designed for image classification. It might be sub-optimal because of the gap between the tasks of image classification and object detection. In this work, we present DetNAS to use Neural Architecture Search (NAS) for the design of better backbones for object detection. It is non-trivial because detection training typically needs ImageNet pre-training while NAS systems require accuracies on the target detection task as supervisory signals. Based on the technique of one-shot supernet, which contains all possible networks in the search space, we propose a framework for backbone search on object detection. We train the supernet under the typical detector training schedule: ImageNet pre-training and detection fine-tuning. Then, the architecture search is performed on the trained supernet, using the detection task as the guidance. This framework makes NAS on backbones very efficient. In experiments, we show the effectiveness of DetNAS on various detectors, for instance, one-stage RetinaNet and the two-stage FPN. We empirically find that networks searched on object detection shows consistent superiority compared to those searched on ImageNet classification. The resulting architecture achieves superior performance than hand-crafted networks on COCO with much less FLOPs complexity. ![pipeline](/docs/en/imgs/model_zoo/detnas/pipeline.jpg) -## Citation - -```latex -@article{chen2019detnas, - title={Detnas: Backbone search for object detection}, - author={Chen, Yukang and Yang, Tong and Zhang, Xiangyu and Meng, Gaofeng and Xiao, Xinyu and Sun, Jian}, - journal={Advances in Neural Information Processing Systems}, - volume={32}, - pages={6642--6652}, - year={2019} -} -``` - -## Results and models -|Dataset| Supernet | Subnet |Params(M)| Flops(G) | mAP | Config | Download | Remarks| -|:---------------:|:---------------:|:-----------:|:-----------:|:-----------:|:--------------:|:------:|:--------:|:--------:| -|COCO| FRCNN-ShuffleNetV2| [mutable](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20211222-67fea61f_mutable_cfg.yaml?versionId=CAEQHxiBgMDU3taI7xciIDUzMmM4MTg4YTgwZDRhYjY4NjA3M2NkZDA0NWExNmY1) | 3.35(backbone)|0.34(backbone) | 37.5 |[config](./detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco.py)|[pretrain](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_shufflenetv2_8xb128_in1k_acc-74.08_20211223-92e9b66a.pth?versionId=CAEQHxiBgICBxuuL7xciIGEyNzZkZmRmZmM5NzRjNDViOTNjOWZkNjk0OWYyYTdm) |[model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20211222-67fea61f.pth?versionId=CAEQHxiBgIDd3taI7xciIDIxYmUzMDE4ZmZmMjQ4ZGNiNzI1YjcxOGM4OGM5NDZl) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20211222-67fea61f.log.json?versionId=CAEQHxiBgMCSq9mM7xciIDViODRmMDE1Yjk1MDQwMTViMDBmYzZlMjg0OTJjYTlh)|MMRazor searched - - -**Note**: -1. The experiment settings of DetNAS are similar with SPOS's, and our training dataset is COCO2017 rather than COCO2014. -2. We also retrained official subnet with same experiment settings, the final result is 36.9 -## Getting Started +## Introduction ### Step 1: Supernet pre-training on ImageNet ```bash python ./tools/mmcls/train_mmcls.py \ @@ -68,3 +48,27 @@ python ./tools/mmdet/train_mmdet.py \ --work-dir $WORK_DIR \ --cfg-options algorithm.mutable_cfg=$STEP3_SUBNET_YAML load_from=$STEP4_CKPT ``` + +## Results and models +|Dataset| Supernet | Subnet |Params(M)| Flops(G) | mAP | Config | Download | Remarks| +|:---------------:|:---------------:|:-----------:|:-----------:|:-----------:|:--------------:|:------:|:--------:|:--------:| +|COCO| FRCNN-ShuffleNetV2| [mutable](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20211222-67fea61f_mutable_cfg.yaml?versionId=CAEQHxiBgMDU3taI7xciIDUzMmM4MTg4YTgwZDRhYjY4NjA3M2NkZDA0NWExNmY1) | 3.35(backbone)|0.34(backbone) | 37.5 |[config](./detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco.py)|[pretrain](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_shufflenetv2_8xb128_in1k_acc-74.08_20211223-92e9b66a.pth?versionId=CAEQHxiBgICBxuuL7xciIGEyNzZkZmRmZmM5NzRjNDViOTNjOWZkNjk0OWYyYTdm) |[model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20211222-67fea61f.pth?versionId=CAEQHxiBgIDd3taI7xciIDIxYmUzMDE4ZmZmMjQ4ZGNiNzI1YjcxOGM4OGM5NDZl) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20211222-67fea61f.log.json?versionId=CAEQHxiBgMCSq9mM7xciIDViODRmMDE1Yjk1MDQwMTViMDBmYzZlMjg0OTJjYTlh)|MMRazor searched + + +**Note**: +1. The experiment settings of DetNAS are similar with SPOS's, and our training dataset is COCO2017 rather than COCO2014. +2. We also retrained official subnet with same experiment settings, the final result is 36.9 + + +## Citation + +```latex +@article{chen2019detnas, + title={Detnas: Backbone search for object detection}, + author={Chen, Yukang and Yang, Tong and Zhang, Xiangyu and Meng, Gaofeng and Xiao, Xinyu and Sun, Jian}, + journal={Advances in Neural Information Processing Systems}, + volume={32}, + pages={6642--6652}, + year={2019} +} +``` diff --git a/configs/nas/detnas/detnas_evolution_search_frcnn_shufflenetv2_fpn_coco.py b/configs/nas/detnas/detnas_evolution_search_frcnn_shufflenetv2_fpn_coco.py index df00b04ce..894fad85a 100644 --- a/configs/nas/detnas/detnas_evolution_search_frcnn_shufflenetv2_fpn_coco.py +++ b/configs/nas/detnas/detnas_evolution_search_frcnn_shufflenetv2_fpn_coco.py @@ -5,6 +5,8 @@ workers_per_gpu=8, ) +algorithm = dict(bn_training_mode=True) + searcher = dict( type='EvolutionSearcher', metrics='bbox', diff --git a/configs/nas/detnas/detnas_supernet_frcnn_shufflenetv2_fpn_1x_coco.py b/configs/nas/detnas/detnas_supernet_frcnn_shufflenetv2_fpn_1x_coco.py index f0720ea29..95f432441 100644 --- a/configs/nas/detnas/detnas_supernet_frcnn_shufflenetv2_fpn_1x_coco.py +++ b/configs/nas/detnas/detnas_supernet_frcnn_shufflenetv2_fpn_1x_coco.py @@ -4,7 +4,7 @@ '../../_base_/mmdet_runtime.py' ] -norm_cfg = dict(type='BN', requires_grad=True) +norm_cfg = dict(type='SyncBN', requires_grad=True) model = dict( type='mmdet.FasterRCNN', backbone=dict( diff --git a/configs/nas/detnas/metafile.yml b/configs/nas/detnas/metafile.yml new file mode 100644 index 000000000..7a9302782 --- /dev/null +++ b/configs/nas/detnas/metafile.yml @@ -0,0 +1,30 @@ +Collections: + - Name: DetNAS + Metadata: + Training Data: + - ImageNet-1k + - COCO + Paper: + URL: https://arxiv.org/abs/1903.10979 + Title: DetNAS:Backbone Search for Object Detection + README: configs/nas/detnas/README.md + Code: + URL: https://github.com/open-mmlab/mmrazor/blob/v0.1.0/mmrazor/models/algorithms/detnas.py + Version: v0.1.0 + Converted From: + Code: https://github.com/megvii-model/DetNAS +Models: + - Name: detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco + In Collection: DetNAS + Metadata: + FLOPs(Backbone): 340 MB + Params(Backbone): 3.35 MB + Supernet: FRCNN-ShuffleNetV2 + Mutable: https://download.openmmlab.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-454627be_mutable_cfg.yaml + Results: + - Task: Object Detection + Dataset: COCO + Metrics: + box AP: 37.5 + Config: configs/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/nas/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20211222-67fea61f_mutable_cfg.yaml diff --git a/configs/nas/spos/README.md b/configs/nas/spos/README.md index 88821c078..68075cf88 100644 --- a/configs/nas/spos/README.md +++ b/configs/nas/spos/README.md @@ -1,5 +1,8 @@ -# SINGLE PATH ONE-SHOT NEURAL ARCHITECTURE SEARCH WITH UNIFORM SAMPLING +# SPOS +> [Single Path One-Shot Neural Architecture Search with Uniform Sampling](https://arxiv.org/abs/1904.00420) + + ## Abstract @@ -8,36 +11,18 @@ Comprehensive experiments verify that our approach is flexible and effective. It ![pipeline](/docs/en/imgs/model_zoo/spos/pipeline.jpg) -## Citation - -```latex -@inproceedings{guo2020single, - title={Single path one-shot neural architecture search with uniform sampling}, - author={Guo, Zichao and Zhang, Xiangyu and Mu, Haoyuan and Heng, Wen and Liu, Zechun and Wei, Yichen and Sun, Jian}, - booktitle={European Conference on Computer Vision}, - pages={544--560}, - year={2020}, - organization={Springer} -} -``` - -## Results and models -|Dataset| Supernet | Subnet | Params(M) | Flops(G) | Top-1 (%) | Top-5 (%) | Config | Download | Remarks | -|:---------------------:|:---------------------:|:------:|:---------:|:--------:|:---------:|:---------:|:------:|:---------|:---------:| -|ImageNet| ShuffleNetV2 |[mutable](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-454627be_mutable_cfg.yaml?versionId=CAEQHxiBgICw5b6I7xciIGY5MjVmNWFhY2U5MjQzN2M4NDViYzI2YWRmYWE1YzQx)| 3.35 | 0.33 | 73.87 | 91.6 |[config](./spos_subnet_shufflenetv2_8xb128_in1k.py)|[model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-1f0a0b4d.pth?versionId=CAEQHxiBgIDK5b6I7xciIDM1YjIwZjQxN2UyMDRjYjA5YTM5NTBlMGNhMTdkNjI2) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-1f0a0b4d.log.json?versionId=CAEQHxiBgIDr9cuL7xciIDBmOTZiZGUyYjRiMDQ5NzhhZjY0NWUxYmUzNDlmNTg5)| MMRazor searched +## Introduction -**Note**: -1. There are some small differences in our experiment in order to be consistent with other repos in OpenMMLab. For example, -normalize images in data preprocessing; resize by cv2 rather than PIL in training; dropout is not used in network. -2. We also retrain the subnet reported in paper with their official code, Top-1 is 73.6 and Top-5 is 91.6 -## Getting Started ### Supernet pre-training on ImageNet + ```bash python ./tools/mmcls/train_mmcls.py \ configs/nas/spos/spos_supernet_shufflenetv2_8xb128_in1k.py \ --work-dir $WORK_DIR ``` + ### Search for subnet on the trained supernet + ```bash python ./tools/mmcls/search_mmcls.py \ configs/nas/spos/spos_evolution_search_shufflenetv2_8xb2048_in1k.py \ @@ -46,9 +31,37 @@ python ./tools/mmcls/search_mmcls.py \ ``` ### Subnet retraining on ImageNet + ```bash python ./tools/mmcls/train_mmcls.py \ configs/nas/spos/spos_subnet_shufflenetv2_8xb128_in1k.py \ --work-dir $WORK_DIR \ --cfg-options algorithm.mutable_cfg=$STEP2_SUBNET_YAML ``` + +## Results and models + +| Dataset | Supernet | Subnet | Params(M) | Flops(G) | Top-1 (%) | Top-5 (%) | Config | Download | Remarks | +| :------: |:----------------------:| :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-------: | :------: | :-------: | :-------: | :----------------------------------------------: |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------:| +| ImageNet | ShuffleNetV2 | [mutable](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-454627be_mutable_cfg.yaml?versionId=CAEQHxiBgICw5b6I7xciIGY5MjVmNWFhY2U5MjQzN2M4NDViYzI2YWRmYWE1YzQx) | 3.35 | 0.33 | 73.87 | 91.6 | [config](./spos_subnet_shufflenetv2_8xb128_in1k.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-1f0a0b4d.pth?versionId=CAEQHxiBgIDK5b6I7xciIDM1YjIwZjQxN2UyMDRjYjA5YTM5NTBlMGNhMTdkNjI2) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-1f0a0b4d.log.json?versionId=CAEQHxiBgIDr9cuL7xciIDBmOTZiZGUyYjRiMDQ5NzhhZjY0NWUxYmUzNDlmNTg5) | MMRazor searched | +| ImageNet | MobileNet-ProxylessGPU | [mutable](https://download.openmmlab.com/mmrazor/v0.1/nas/spos/spos_mobilenet_subnet/spos_angelnas_flops_0.49G_acc_75.98_20220307-54f4698f_mutable_cfg.yaml) | 5.94 | 0.49* | 75.98 | 92.77 | [config](./spos_mobilenet_for_check_ckpt_from_anglenas.py) | | [AngleNAS](https://github.com/megvii-model/AngleNAS) searched | + +**Note**: + +1. There **might be(not all the case)** some small differences in our experiment in order to be consistent with other repos in OpenMMLab. For example, + normalize images in data preprocessing; resize by cv2 rather than PIL in training; dropout is not used in network. **Please refer to corresponding config for details.** +2. For *ShuffleNetV2*, we retrain the subnet reported in paper with their official code, Top-1 is 73.6 and Top-5 is 91.6. +2. For *AngleNAS searched MobileNet-ProxylessGPU*, we obtain params and FLOPs using [this script](/tools/misc/get_flops.py), which may be different from [AngleNAS](https://github.com/megvii-model/AngleNAS#searched-models-with-abs). + +## Citation + +```latex +@inproceedings{guo2020single, + title={Single path one-shot neural architecture search with uniform sampling}, + author={Guo, Zichao and Zhang, Xiangyu and Mu, Haoyuan and Heng, Wen and Liu, Zechun and Wei, Yichen and Sun, Jian}, + booktitle={European Conference on Computer Vision}, + pages={544--560}, + year={2020}, + organization={Springer} +} +``` diff --git a/configs/nas/spos/SPOS_MOBILENET_490M_FROM_ANGELNAS.yaml b/configs/nas/spos/SPOS_MOBILENET_490M_FROM_ANGELNAS.yaml new file mode 100644 index 000000000..154cc6704 --- /dev/null +++ b/configs/nas/spos/SPOS_MOBILENET_490M_FROM_ANGELNAS.yaml @@ -0,0 +1,66 @@ +stage_0_block_0: + chosen: + - mb_k3e1 +stage_1_block_0: + chosen: + - mb_k5e3 +stage_1_block_1: + chosen: + - mb_k5e3 +stage_1_block_2: + chosen: + - identity +stage_1_block_3: + chosen: + - mb_k3e3 +stage_2_block_0: + chosen: + - mb_k3e3 +stage_2_block_1: + chosen: + - identity +stage_2_block_2: + chosen: + - identity +stage_2_block_3: + chosen: + - mb_k3e3 +stage_3_block_0: + chosen: + - mb_k7e6 +stage_3_block_1: + chosen: + - identity +stage_3_block_2: + chosen: + - mb_k7e3 +stage_3_block_3: + chosen: + - mb_k7e3 +stage_4_block_0: + chosen: + - mb_k3e3 +stage_4_block_1: + chosen: + - mb_k3e3 +stage_4_block_2: + chosen: + - mb_k7e3 +stage_4_block_3: + chosen: + - mb_k5e3 +stage_5_block_0: + chosen: + - mb_k5e6 +stage_5_block_1: + chosen: + - mb_k7e3 +stage_5_block_2: + chosen: + - mb_k7e3 +stage_5_block_3: + chosen: + - mb_k7e3 +stage_6_block_0: + chosen: + - mb_k5e6 diff --git a/configs/nas/spos/metafile.yml b/configs/nas/spos/metafile.yml new file mode 100644 index 000000000..5c13d5741 --- /dev/null +++ b/configs/nas/spos/metafile.yml @@ -0,0 +1,29 @@ +Collections: + - Name: SPOS + Metadata: + Training Data: + - ImageNet-1k + Paper: + URL: https://arxiv.org/abs/1904.00420 + Title: Single Path One-Shot Neural Architecture Search with Uniform Sampling + README: configs/nas/spos/README.md + Code: + URL: https://github.com/open-mmlab/mmrazor/blob/v0.1.0/mmrazor/models/algorithms/spos.py + Version: v0.1.0 + Converted From: + Code: https://github.com/megvii-model/SinglePathOneShot +Models: + - Name: spos_subnet_shufflenetv2_8xb128_in1k + In Collection: SPOS + Metadata: + FLOPs: 330 MB + Supernet: ShuffleNetV2 + Mutable: https://download.openmmlab.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-454627be_mutable_cfg.yaml + Results: + - Task: Image Classification + Dataset: ImageNet-1k + Metrics: + Top 1 Accuracy: 73.87 + Top 5 Accuracy: 91.60 + Config: configs/nas/spos/spos_subnet_shufflenetv2_8xb128_in1k.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-1f0a0b4d.pth diff --git a/configs/nas/spos/spos_evolution_search_mobilenet_proxyless_gpu_flops465_8xb512_in1k.py b/configs/nas/spos/spos_evolution_search_mobilenet_proxyless_gpu_flops465_8xb512_in1k.py new file mode 100644 index 000000000..a37fafc00 --- /dev/null +++ b/configs/nas/spos/spos_evolution_search_mobilenet_proxyless_gpu_flops465_8xb512_in1k.py @@ -0,0 +1,20 @@ +_base_ = ['./spos_supernet_mobilenet_proxyless_gpu_8xb128_in1k.py'] + +data = dict( + samples_per_gpu=512, + workers_per_gpu=16, +) + +algorithm = dict(bn_training_mode=True) + +searcher = dict( + type='EvolutionSearcher', + candidate_pool_size=50, + candidate_top_k=10, + constraints=dict(flops=465 * 1e6), + metrics='accuracy', + score_key='accuracy_top-1', + max_epoch=20, + num_mutation=25, + num_crossover=25, + mutate_prob=0.1) diff --git a/configs/nas/spos/spos_mobilenet_for_check_ckpt_from_anglenas.py b/configs/nas/spos/spos_mobilenet_for_check_ckpt_from_anglenas.py new file mode 100644 index 000000000..3423abd43 --- /dev/null +++ b/configs/nas/spos/spos_mobilenet_for_check_ckpt_from_anglenas.py @@ -0,0 +1,27 @@ +_base_ = [ + './spos_subnet_mobilenet_proxyless_gpu_8xb128_in1k.py', +] + +img_norm_cfg = dict(mean=[0., 0., 0.], std=[1., 1., 1.], to_rgb=False) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='RandomResizedCrop', size=224), + dict(type='ColorJitter', brightness=0.4, contrast=0.4, saturation=0.4), + dict(type='RandomFlip', flip_prob=0.5, direction='horizontal'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='ToTensor', keys=['gt_label']), + dict(type='Collect', keys=['img', 'gt_label']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='Resize', size=(256, -1)), + dict(type='CenterCrop', crop_size=224), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) +] +data = dict( + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) diff --git a/configs/nas/spos/spos_subnet_mobilenet_proxyless_gpu_8xb128_in1k.py b/configs/nas/spos/spos_subnet_mobilenet_proxyless_gpu_8xb128_in1k.py new file mode 100644 index 000000000..198a9c053 --- /dev/null +++ b/configs/nas/spos/spos_subnet_mobilenet_proxyless_gpu_8xb128_in1k.py @@ -0,0 +1,10 @@ +_base_ = [ + './spos_supernet_mobilenet_proxyless_gpu_8xb128_in1k.py', +] + +algorithm = dict(retraining=True) +evaluation = dict(interval=10000, metric='accuracy') +checkpoint_config = dict(interval=30000) + +runner = dict(max_iters=300000) +find_unused_parameters = False diff --git a/configs/nas/spos/spos_supernet_mobilenet_proxyless_gpu_8xb128_in1k.py b/configs/nas/spos/spos_supernet_mobilenet_proxyless_gpu_8xb128_in1k.py new file mode 100644 index 000000000..262c1c647 --- /dev/null +++ b/configs/nas/spos/spos_supernet_mobilenet_proxyless_gpu_8xb128_in1k.py @@ -0,0 +1,101 @@ +_base_ = [ + '../../_base_/datasets/mmcls/imagenet_bs128_colorjittor.py', + '../../_base_/schedules/mmcls/imagenet_bs1024_spos.py', + '../../_base_/mmcls_runtime.py' +] +norm_cfg = dict(type='BN') +model = dict( + type='mmcls.ImageClassifier', + backbone=dict( + type='SearchableMobileNet', + first_channels=40, + last_channels=1728, + widen_factor=1.0, + norm_cfg=norm_cfg, + arch_setting_type='proxyless_gpu'), + neck=dict(type='GlobalAveragePooling'), + head=dict( + type='LinearClsHead', + num_classes=1000, + in_channels=1728, + loss=dict( + type='LabelSmoothLoss', + num_classes=1000, + label_smooth_val=0.1, + mode='original', + loss_weight=1.0), + topk=(1, 5), + ), +) + +mutator = dict( + type='OneShotMutator', + placeholder_mapping=dict( + searchable_blocks=dict( + type='OneShotOP', + choices=dict( + mb_k3e3=dict( + type='MBBlock', + kernel_size=3, + expand_ratio=3, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU6')), + mb_k5e3=dict( + type='MBBlock', + kernel_size=5, + expand_ratio=3, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU6')), + mb_k7e3=dict( + type='MBBlock', + kernel_size=7, + expand_ratio=3, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU6')), + mb_k3e6=dict( + type='MBBlock', + kernel_size=3, + expand_ratio=6, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU6')), + mb_k5e6=dict( + type='MBBlock', + kernel_size=5, + expand_ratio=6, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU6')), + mb_k7e6=dict( + type='MBBlock', + kernel_size=7, + expand_ratio=6, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU6')), + identity=dict(type='Identity'))), + first_blocks=dict( + type='OneShotOP', + choices=dict( + mb_k3e1=dict( + type='MBBlock', + kernel_size=3, + expand_ratio=1, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU6')), )))) + +algorithm = dict( + type='SPOS', + architecture=dict( + type='MMClsArchitecture', + model=model, + ), + mutator=mutator, + distiller=None, + retraining=False, +) + +runner = dict(max_iters=150000) +evaluation = dict(interval=10000, metric='accuracy') + +# checkpoint saving +checkpoint_config = dict(interval=30000) + +find_unused_parameters = True diff --git a/configs/pruning/autoslim/README.md b/configs/pruning/autoslim/README.md index a58f718ff..84d3b7e5f 100644 --- a/configs/pruning/autoslim/README.md +++ b/configs/pruning/autoslim/README.md @@ -1,4 +1,7 @@ -# AUTOSLIM: TOWARDS ONE-SHOT ARCHITECTURE SEARCH FOR CHANNEL NUMBERS +# AutoSlim +> [AutoSlim: Towards One-Shot Architecture Search for Channel Numbers](https://arxiv.org/abs/1903.11728) + + ## Abstract @@ -7,34 +10,8 @@ Notably, by setting optimized channel numbers, our AutoSlim-MobileNet-v2 at 305M ![pipeline](/docs/en/imgs/model_zoo/autoslim/pipeline.png) -## Citation - -```latex -@article{yu2019autoslim, - title={Autoslim: Towards one-shot architecture search for channel numbers}, - author={Yu, Jiahui and Huang, Thomas}, - journal={arXiv preprint arXiv:1903.11728}, - year={2019} -} -``` - -## Results and models -### Subnet retrain -| Supernet | Params(M) | Flops(G) | Top-1 (%) | Top-5 (%) | Config | Download | Subnet | Remark | -| :----------------- | :-------: | -------: | :-------: | :-------: | :----: | :------: | :-------------: | :----: | -| MobileNet v2(x1.5) | 6.5 | 0.53 | 74.23 | 91.74 | [config](./autoslim_mbv2_subnet_8xb256_in1k.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.53M_acc-74.23_20211222-e5208bbd.pth?versionId=CAEQHxiBgICYsIaI7xciIDE1MGIxM2Q5NDk1NjRlOTFiMjgwOTRmYzJlMDBmZDY0) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1kautoslim_mbv2_subnet_8xb256_in1k_paper_channel_cfg.log.json?versionId=CAEQHxiBgMCjj9SL7xciIDFmYmM4NTExZmIzNjRmNmQ4MmMyZWI4YzJmMmM2MDdl) | [channel](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.53M_acc-74.23_20211222-e5208bbd_channel_cfg.yaml?versionId=CAEQHxiBgMDwr4aI7xciIDQ2MmRhMDFhNGMyODQyYmU5ZTIyOTcxMmRlN2RmYjg2) | official channel cfg | -| MobileNet v2(x1.5) | 5.77 | 0.32 | 72.73 | 90.83 | [config](./autoslim_mbv2_subnet_8xb256_in1k.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.32M_acc-72.73_20211222-b5b0b33c.pth?versionId=CAEQHxiBgMCasIaI7xciIDEzN2FkZjZkNWMwYjRiOTg5NTY0MzY0ODk5ODE2N2Yz) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1kautoslim_mbv2_subnet_8xb256_in1k_paper_channel_cfg.log.json?versionId=CAEQHxiBgMCjj9SL7xciIDFmYmM4NTExZmIzNjRmNmQ4MmMyZWI4YzJmMmM2MDdl) | [channel](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.32M_acc-72.73_20211222-b5b0b33c_channel_cfg.yaml?versionId=CAEQHxiCgMDwr4aI7xciIDhjMmUzZjlmZTJjODQzMDRhMmQxMzkyM2MwOTZhNjE3) | official channel cfg | -| MobileNet v2(x1.5) | 4.13 |0.22 | 71.39 | 90.08 | [config](./autoslim_mbv2_subnet_8xb256_in1k.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.22M_acc-71.39_20211222-43117c7b.pth?versionId=CAEQHxiBgICRsIaI7xciIDVlY2MxMTkwZjg0ODQ3M2I5NTJmYjFiNDk1MDEwNjAy) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1kautoslim_mbv2_subnet_8xb256_in1k_paper_channel_cfg.log.json?versionId=CAEQHxiBgMCjj9SL7xciIDFmYmM4NTExZmIzNjRmNmQ4MmMyZWI4YzJmMmM2MDdl) | [channel](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.22M_acc-71.39_20211222-43117c7b_channel_cfg.yaml.?versionId=CAEQHxiBgIDzr4aI7xciIDViNGY0ZDA1ODkxZTRkMGFhNTg2M2FlZmQyZTFiMDgx) | official channel cfg | - -Note that we ran the official code and the Top-1 Acc of the models with official -channel cfg are 73.8%, 72.5% and 71.1%. And there are 3 differences between our -implementation and the official one. -1. The implementation of Label Smooth is slightly different. -2. Lighting is not used in our data pipeline. (Lighting is a kind of data -augmentation which adjust images lighting using AlexNet-style PCA jitter.) -3. We do not recalibrating BN statistics after training. -## Getting Started +## Introduction ### Supernet pre-training on ImageNet 
 python ./tools/mmcls/train_mmcls.py \
@@ -73,3 +50,32 @@ python ./tools/mmcls/test_mmcls.py \
   your_splitted_checkpoint_path --metrics accuracy \
   --cfg-options algorithm.channel_cfg=configs/pruning/autoslim/AUTOSLIM_MBV2_530M_OFFICIAL.yaml
+## Results and models +### Subnet retrain +| Supernet | Params(M) | Flops(G) | Top-1 (%) | Top-5 (%) | Config | Download | Subnet | Remark | +| :----------------- | :-------: | -------: | :-------: | :-------: | :----: | :------: | :-------------: | :----: | +| MobileNet v2(x1.5) | 6.5 | 0.53 | 74.23 | 91.74 | [config](./autoslim_mbv2_subnet_8xb256_in1k.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.53M_acc-74.23_20211222-e5208bbd.pth?versionId=CAEQHxiBgICYsIaI7xciIDE1MGIxM2Q5NDk1NjRlOTFiMjgwOTRmYzJlMDBmZDY0) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1kautoslim_mbv2_subnet_8xb256_in1k_paper_channel_cfg.log.json?versionId=CAEQHxiBgMCjj9SL7xciIDFmYmM4NTExZmIzNjRmNmQ4MmMyZWI4YzJmMmM2MDdl) | [channel](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.53M_acc-74.23_20211222-e5208bbd_channel_cfg.yaml?versionId=CAEQHxiBgMDwr4aI7xciIDQ2MmRhMDFhNGMyODQyYmU5ZTIyOTcxMmRlN2RmYjg2) | official channel cfg | +| MobileNet v2(x1.5) | 5.77 | 0.32 | 72.73 | 90.83 | [config](./autoslim_mbv2_subnet_8xb256_in1k.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.32M_acc-72.73_20211222-b5b0b33c.pth?versionId=CAEQHxiBgMCasIaI7xciIDEzN2FkZjZkNWMwYjRiOTg5NTY0MzY0ODk5ODE2N2Yz) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1kautoslim_mbv2_subnet_8xb256_in1k_paper_channel_cfg.log.json?versionId=CAEQHxiBgMCjj9SL7xciIDFmYmM4NTExZmIzNjRmNmQ4MmMyZWI4YzJmMmM2MDdl) | [channel](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.32M_acc-72.73_20211222-b5b0b33c_channel_cfg.yaml?versionId=CAEQHxiCgMDwr4aI7xciIDhjMmUzZjlmZTJjODQzMDRhMmQxMzkyM2MwOTZhNjE3) | official channel cfg | +| MobileNet v2(x1.5) | 4.13 |0.22 | 71.39 | 90.08 | [config](./autoslim_mbv2_subnet_8xb256_in1k.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.22M_acc-71.39_20211222-43117c7b.pth?versionId=CAEQHxiBgICRsIaI7xciIDVlY2MxMTkwZjg0ODQ3M2I5NTJmYjFiNDk1MDEwNjAy) | [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1kautoslim_mbv2_subnet_8xb256_in1k_paper_channel_cfg.log.json?versionId=CAEQHxiBgMCjj9SL7xciIDFmYmM4NTExZmIzNjRmNmQ4MmMyZWI4YzJmMmM2MDdl) | [channel](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.22M_acc-71.39_20211222-43117c7b_channel_cfg.yaml.?versionId=CAEQHxiBgIDzr4aI7xciIDViNGY0ZDA1ODkxZTRkMGFhNTg2M2FlZmQyZTFiMDgx) | official channel cfg | + +Note that we ran the official code and the Top-1 Acc of the models with official +channel cfg are 73.8%, 72.5% and 71.1%. And there are 3 differences between our +implementation and the official one. +1. The implementation of Label Smooth is slightly different. +2. Lighting is not used in our data pipeline. (Lighting is a kind of data +augmentation which adjust images lighting using AlexNet-style PCA jitter.) +3. We do not recalibrating BN statistics after training. + + + + +## Citation + +```latex +@article{yu2019autoslim, + title={Autoslim: Towards one-shot architecture search for channel numbers}, + author={Yu, Jiahui and Huang, Thomas}, + journal={arXiv preprint arXiv:1903.11728}, + year={2019} +} +``` diff --git a/configs/pruning/autoslim/metafile.yml b/configs/pruning/autoslim/metafile.yml new file mode 100644 index 000000000..de82d8d78 --- /dev/null +++ b/configs/pruning/autoslim/metafile.yml @@ -0,0 +1,60 @@ +Collections: + - Name: AutoSlim + Metadata: + Training Data: + - ImageNet-1k + Paper: + URL: https://arxiv.org/abs/1903.11728 + Title: AutoSlim:Towards One-Shot Architecture Search for Channel Numbers + README: configs/nas/spos/README.md + Code: + URL: https://github.com/open-mmlab/mmrazor/blob/v0.1.0/mmrazor/models/algorithms/autoslim.py + Version: v0.1.0 + Converted From: + Code: https://github.com/JiahuiYu/slimmable_networks +Models: + - Name: autoslim_mbv2_subnet_8xb256_in1k + In Collection: AutoSlim + Metadata: + Flops(G): 0.53 + Params(M): 6.5 + Supernet: MobileNet v2(x1.5) + Channel: https://download.openmmlab.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.53M_acc-74.23_20211222-e5208bbd_channel_cfg.yaml + Results: + - Task: Image Classification + Dataset: ImageNet-1k + Metrics: + Top 1 Accuracy: 74.23 + Top 5 Accuracy: 91.74 + Config: configs/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.53M_acc-74.23_20211222-e5208bbd.pth + - Name: autoslim_mbv2_subnet_8xb256_in1k + In Collection: AutoSlim + Metadata: + Flops(G): 0.32 + Params(M): 5.77 + Supernet: MobileNet v2(x1.5) + Channel: https://download.openmmlab.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.32M_acc-72.73_20211222-b5b0b33c_channel_cfg.yaml + Results: + - Task: Image Classification + Dataset: ImageNet-1k + Metrics: + Top 1 Accuracy: 72.73 + Top 5 Accuracy: 90.83 + Config: configs/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.32M_acc-72.73_20211222-b5b0b33c.pth + - Name: autoslim_mbv2_subnet_8xb256_in1k + In Collection: AutoSlim + Metadata: + Flops(G): 0.22 + Params(M): 4.13 + Supernet: MobileNet v2(x1.5) + Channel: https://download.openmmlab.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.22M_acc-71.39_20211222-43117c7b_channel_cfg.yaml.? + Results: + - Task: Image Classification + Dataset: ImageNet-1k + Metrics: + Top 1 Accuracy: 74.23 + Top 5 Accuracy: 91.74 + Config: configs/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py + Weights: https://download.openmmlab.com/mmrazor/v0.1/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k/autoslim_mbv2_subnet_8xb256_in1k_flops-0.22M_acc-71.39_20211222-43117c7b.pth diff --git a/mmrazor/apis/mmdet/train.py b/mmrazor/apis/mmdet/train.py index 593a91dd4..4f606f472 100644 --- a/mmrazor/apis/mmdet/train.py +++ b/mmrazor/apis/mmdet/train.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import random import warnings diff --git a/mmrazor/core/builder.py b/mmrazor/core/builder.py index bbffdb916..b72c04ac7 100644 --- a/mmrazor/core/builder.py +++ b/mmrazor/core/builder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.utils import Registry, build_from_cfg SEARCHERS = Registry('search') diff --git a/mmrazor/core/hooks/__init__.py b/mmrazor/core/hooks/__init__.py index 984f76aaa..8edec6976 100644 --- a/mmrazor/core/hooks/__init__.py +++ b/mmrazor/core/hooks/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .drop_path_prob import DropPathProbHook from .sampler_seed import DistSamplerSeedHook from .search_subnet import SearchSubnetHook diff --git a/mmrazor/core/optimizer/__init__.py b/mmrazor/core/optimizer/__init__.py index ea6b36f64..e1c477dd1 100644 --- a/mmrazor/core/optimizer/__init__.py +++ b/mmrazor/core/optimizer/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .builder import build_optimizers __all__ = ['build_optimizers'] diff --git a/mmrazor/core/optimizer/builder.py b/mmrazor/core/optimizer/builder.py index f2b30677a..115b1df6a 100644 --- a/mmrazor/core/optimizer/builder.py +++ b/mmrazor/core/optimizer/builder.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcv.runner import build_optimizer diff --git a/mmrazor/core/runners/__init__.py b/mmrazor/core/runners/__init__.py index 2fa16854e..502559cb0 100644 --- a/mmrazor/core/runners/__init__.py +++ b/mmrazor/core/runners/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .epoch_based_runner import MultiLoaderEpochBasedRunner from .iter_based_runner import MultiLoaderIterBasedRunner diff --git a/mmrazor/core/searcher/__init__.py b/mmrazor/core/searcher/__init__.py index 58753947e..b4be5b22b 100644 --- a/mmrazor/core/searcher/__init__.py +++ b/mmrazor/core/searcher/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .evolution_search import EvolutionSearcher from .greedy_search import GreedySearcher diff --git a/mmrazor/core/utils/lr.py b/mmrazor/core/utils/lr.py index 9acf101cf..e731681da 100644 --- a/mmrazor/core/utils/lr.py +++ b/mmrazor/core/utils/lr.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. def set_lr(runner, lr_groups, freeze_optimizers=[]): """Set specified learning rate in optimizer.""" if isinstance(runner.optimizer, dict): diff --git a/mmrazor/datasets/__init__.py b/mmrazor/datasets/__init__.py index 921f045a2..ff514395d 100644 --- a/mmrazor/datasets/__init__.py +++ b/mmrazor/datasets/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from .utils import split_dataset __all__ = ['split_dataset'] diff --git a/mmrazor/datasets/utils.py b/mmrazor/datasets/utils.py index 08d0dec26..2a0852700 100644 --- a/mmrazor/datasets/utils.py +++ b/mmrazor/datasets/utils.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from torch.utils.data import random_split diff --git a/mmrazor/models/architectures/components/backbones/__init__.py b/mmrazor/models/architectures/components/backbones/__init__.py index 63d8372c9..6fd18ba88 100644 --- a/mmrazor/models/architectures/components/backbones/__init__.py +++ b/mmrazor/models/architectures/components/backbones/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. from .darts_backbone import DartsBackbone +from .searchable_mobilenet import SearchableMobileNet from .searchable_shufflenet_v2 import SearchableShuffleNetV2 -__all__ = ['DartsBackbone', 'SearchableShuffleNetV2'] +__all__ = ['DartsBackbone', 'SearchableShuffleNetV2', 'SearchableMobileNet'] diff --git a/mmrazor/models/architectures/components/backbones/darts_backbone.py b/mmrazor/models/architectures/components/backbones/darts_backbone.py index a08dbdb2f..79091dd36 100644 --- a/mmrazor/models/architectures/components/backbones/darts_backbone.py +++ b/mmrazor/models/architectures/components/backbones/darts_backbone.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import copy import torch diff --git a/mmrazor/models/architectures/components/backbones/searchable_mobilenet.py b/mmrazor/models/architectures/components/backbones/searchable_mobilenet.py new file mode 100644 index 000000000..cd24e5598 --- /dev/null +++ b/mmrazor/models/architectures/components/backbones/searchable_mobilenet.py @@ -0,0 +1,214 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn +from mmcls.models.backbones.base_backbone import BaseBackbone +from mmcls.models.builder import BACKBONES +from mmcls.models.utils import make_divisible +from mmcv.cnn import ConvModule +from torch.nn.modules.batchnorm import _BatchNorm + +from ...utils import Placeholder + + +@BACKBONES.register_module() +class SearchableMobileNet(BaseBackbone): + """Searchable MobileNet backbone. + + Args: + first_channels (int): Channel width of first ConvModule. Default: 32. + last_channels (int): Channel width of last ConvModule. Default: 1200. + widen_factor (float): Width multiplier, multiply number of + channels in each layer by this amount. Default: 1.0. + out_indices (None or Sequence[int]): Output from which stages. + Default: (7, ). + frozen_stages (int): Stages to be frozen (all param fixed). + Default: -1, which means not freezing any parameters. + conv_cfg (dict, optional): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU6'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + arch_setting_type (str): Specify architecture setting. + Default: 'original'. + init_cfg (dict | list[dict]): initialization configuration dict to + define initializer. OpenMMLab has implemented 6 initializers + including ``Constant``, ``Xavier``, ``Normal``, ``Uniform``, + ``Kaiming``, and ``Pretrained``. + """ + + # Parameters to build layers. 3 parameters are needed to construct a + # layer, from left to right: channel, num_blocks, stride. + arch_settings_dict = { + 'original': [[16, 1, 1], [24, 2, 2], [32, 3, 2], [64, 4, 2], + [96, 3, 1], [160, 3, 2], [320, 1, 1]], + 'proxyless_gpu': [[24, 1, 1], [32, 4, 2], [56, 4, 2], [112, 4, 2], + [128, 4, 1], [256, 4, 2], [432, 1, 1]], + } + + def __init__(self, + first_channels=32, + last_channels=1280, + widen_factor=1., + out_indices=(7, ), + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU6'), + norm_eval=False, + with_cp=False, + arch_setting_type='original', + init_cfg=[ + dict(type='Kaiming', layer=['Conv2d']), + dict( + type='Constant', + val=1, + layer=['_BatchNorm', 'GroupNorm']) + ]): + super(SearchableMobileNet, self).__init__(init_cfg) + + arch_settings = self.arch_settings_dict.get(arch_setting_type) + if arch_settings is None: + raise ValueError(f'Expect `arch_setting_type`: ' + f'{list(self.arch_settings_dict.keys())}, ' + f'but got: {arch_setting_type}') + self.arch_settings = arch_settings + self.widen_factor = widen_factor + self.out_indices = out_indices + for index in out_indices: + if index not in range(0, 8): + raise ValueError('the item in out_indices must in ' + f'range(0, 8). But received {index}') + + if frozen_stages not in range(-1, 8): + raise ValueError('frozen_stages must be in range(-1, 8). ' + f'But received {frozen_stages}') + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.norm_eval = norm_eval + self.with_cp = with_cp + + self.in_channels = make_divisible(first_channels * widen_factor, 8) + + self.conv1 = ConvModule( + in_channels=3, + out_channels=self.in_channels, + kernel_size=3, + stride=2, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.layers = [] + + for i, layer_cfg in enumerate(self.arch_settings): + channel, num_blocks, stride = layer_cfg + out_channels = make_divisible(channel * widen_factor, 8) + inverted_res_layer = self.make_layer( + out_channels=out_channels, + num_blocks=num_blocks, + stride=stride, + stage_idx=i) + layer_name = f'layer{i + 1}' + self.add_module(layer_name, inverted_res_layer) + self.layers.append(layer_name) + + if widen_factor > 1.0: + self.out_channel = int(last_channels * widen_factor) + else: + self.out_channel = last_channels + + layer = ConvModule( + in_channels=self.in_channels, + out_channels=self.out_channel, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.add_module('conv2', layer) + self.layers.append('conv2') + + def make_layer(self, out_channels, num_blocks, stride, stage_idx): + """Stack InvertedResidual blocks to build a layer for MobileNetV2. + + Args: + out_channels (int): out_channels of block. + num_blocks (int): number of blocks. + stride (int): stride of the first block. Default: 1 + expand_ratio (int): Expand the number of channels of the + hidden layer in InvertedResidual by this ratio. Default: 6. + """ + layers = [] + for i in range(num_blocks): + if i >= 1: + stride = 1 + # HACK + # do not search first block + if stage_idx == 0: + group = 'first_blocks' + else: + group = 'searchable_blocks' + layers.append( + Placeholder( + group=group, + space_id=f'stage_{stage_idx}_block_{i}', + choice_args=dict( + in_channels=self.in_channels, + out_channels=out_channels, + stride=stride, + ))) + self.in_channels = out_channels + + return nn.Sequential(*layers) + + def forward(self, x): + """Forward computation. + + Args: + x (tensor | tuple[tensor]): x could be a Torch.tensor or a tuple of + Torch.tensor, containing input data for forward computation. + """ + x = self.conv1(x) + + outs = [] + for i, layer_name in enumerate(self.layers): + layer = getattr(self, layer_name) + x = layer(x) + if i in self.out_indices: + outs.append(x) + + return tuple(outs) + + def _freeze_stages(self): + """Freeze params not to update in the specified stages.""" + if self.frozen_stages >= 0: + for param in self.conv1.parameters(): + param.requires_grad = False + for i in range(1, self.frozen_stages + 1): + layer = getattr(self, f'layer{i}') + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + """Set module status before forward computation. + + Args: + mode (bool): Whether it is train_mode or test_mode + """ + super(SearchableMobileNet, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, _BatchNorm): + m.eval() diff --git a/mmrazor/models/architectures/components/heads/no_bias_fc_head.py b/mmrazor/models/architectures/components/heads/no_bias_fc_head.py index 70ad72a77..a30ea8a79 100644 --- a/mmrazor/models/architectures/components/heads/no_bias_fc_head.py +++ b/mmrazor/models/architectures/components/heads/no_bias_fc_head.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. from mmcls.models.builder import HEADS from mmcls.models.heads import LinearClsHead from torch import nn diff --git a/mmrazor/models/distillers/base.py b/mmrazor/models/distillers/base.py index 84039c201..c62294889 100644 --- a/mmrazor/models/distillers/base.py +++ b/mmrazor/models/distillers/base.py @@ -120,7 +120,7 @@ def __init__(self, align_methods=None, **kwargs): self.context_manager = ConversionContext(align_methods) @abstractmethod - def prepare_from_student(self, supernet): + def prepare_from_student(self, student): """Register forward hooks to students and teachers.""" pass diff --git a/mmrazor/models/ops/__init__.py b/mmrazor/models/ops/__init__.py index 7cc85ae81..a3b8c1e0e 100644 --- a/mmrazor/models/ops/__init__.py +++ b/mmrazor/models/ops/__init__.py @@ -1,9 +1,11 @@ # Copyright (c) OpenMMLab. All rights reserved. +from .common import Identity from .darts_series import (DartsDilConv, DartsPoolBN, DartsSepConv, DartsSkipConnect, DartsZero) +from .mobilenet_series import MBBlock from .shufflenet_series import ShuffleBlock, ShuffleXception __all__ = [ 'ShuffleBlock', 'ShuffleXception', 'DartsPoolBN', 'DartsDilConv', - 'DartsSepConv', 'DartsSkipConnect', 'DartsZero' + 'DartsSepConv', 'DartsSkipConnect', 'DartsZero', 'MBBlock', 'Identity' ] diff --git a/mmrazor/models/ops/common.py b/mmrazor/models/ops/common.py new file mode 100644 index 000000000..830c6babf --- /dev/null +++ b/mmrazor/models/ops/common.py @@ -0,0 +1,48 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from mmcv.cnn import ConvModule + +from ..builder import OPS +from .base import BaseOP + + +@OPS.register_module() +class Identity(BaseOP): + """Base class for searchable operations. + + Args: + conv_cfg (dict, optional): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: None. + """ + + def __init__(self, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=None, + **kwargs): + super(Identity, self).__init__(**kwargs) + + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + + if self.stride != 1 or self.in_channels != self.out_channels: + self.downsample = ConvModule( + self.in_channels, + self.out_channels, + kernel_size=1, + stride=self.stride, + padding=0, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + else: + self.downsample = None + + def forward(self, x): + if self.downsample is not None: + x = self.downsample(x) + return x diff --git a/mmrazor/models/ops/darts_series.py b/mmrazor/models/ops/darts_series.py index e2e781f0f..f5d13f2bf 100644 --- a/mmrazor/models/ops/darts_series.py +++ b/mmrazor/models/ops/darts_series.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn from mmcv.cnn import build_norm_layer diff --git a/mmrazor/models/ops/mobilenet_series.py b/mmrazor/models/ops/mobilenet_series.py new file mode 100644 index 000000000..9c63c4e23 --- /dev/null +++ b/mmrazor/models/ops/mobilenet_series.py @@ -0,0 +1,129 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn +import torch.utils.checkpoint as cp +from mmcls.models.utils import SELayer +from mmcv.cnn import ConvModule +from mmcv.cnn.bricks import DropPath + +from ..builder import OPS +from .base import BaseOP + + +@OPS.register_module() +class MBBlock(BaseOP): + """Mobilenet block for Searchable backbone. + + Args: + kernel_size (int): Size of the convolving kernel. + expand_ratio (int): The input channels' expand factor of the depthwise + convolution. + se_cfg (dict, optional): Config dict for se layer. Defaults to None, + which means no se layer. + conv_cfg (dict, optional): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU'). + drop_path_rate (float): stochastic depth rate. Defaults to 0. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, + kernel_size, + expand_ratio, + se_cfg=None, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + drop_path_rate=0., + with_cp=False, + **kwargs): + + super(MBBlock, self).__init__(**kwargs) + self.with_res_shortcut = ( + self.stride == 1 and self.in_channels == self.out_channels) + assert self.stride in [1, 2] + self.kernel_size = kernel_size + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.with_cp = with_cp + self.drop_path = DropPath( + drop_path_rate) if drop_path_rate > 0 else nn.Identity() + self.with_se = se_cfg is not None + self.mid_channels = self.in_channels * expand_ratio + self.with_expand_conv = (self.mid_channels != self.in_channels) + + if self.with_se: + assert isinstance(se_cfg, dict) + + if self.with_expand_conv: + self.expand_conv = ConvModule( + in_channels=self.in_channels, + out_channels=self.mid_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.depthwise_conv = ConvModule( + in_channels=self.mid_channels, + out_channels=self.mid_channels, + kernel_size=kernel_size, + stride=self.stride, + padding=kernel_size // 2, + groups=self.mid_channels, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + if self.with_se: + self.se = SELayer(self.mid_channels, **se_cfg) + self.linear_conv = ConvModule( + in_channels=self.mid_channels, + out_channels=self.out_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + def forward(self, x): + """Forward function. + + Args: + x (torch.Tensor): The input tensor. + Returns: + torch.Tensor: The output tensor. + """ + + def _inner_forward(x): + out = x + + if self.with_expand_conv: + out = self.expand_conv(out) + + out = self.depthwise_conv(out) + + if self.with_se: + out = self.se(out) + + out = self.linear_conv(out) + + if self.with_res_shortcut: + return x + self.drop_path(out) + else: + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out diff --git a/mmrazor/version.py b/mmrazor/version.py index 9e809e870..e7cc0f196 100644 --- a/mmrazor/version.py +++ b/mmrazor/version.py @@ -1,6 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved -__version__ = '0.1.0' +__version__ = '0.2.0' def parse_version_info(version_str): diff --git a/model-index.yml b/model-index.yml new file mode 100644 index 000000000..8d816a206 --- /dev/null +++ b/model-index.yml @@ -0,0 +1,7 @@ +Import: + - configs/distill/cwd/metafile.yml + - configs/distill/wsld/metafile.yml + - configs/nas/darts/metafile.yml + - configs/nas/detnas/metafile.yml + - configs/nas/spos/metafile.yml + - configs/pruning/autoslim/metafile.yml diff --git a/requirements/optional.txt b/requirements/optional.txt index 8d4875354..c63d4ff48 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -1,4 +1,4 @@ -albumentations>=0.3.2 --no-binary imgaug,albumentations +albumentations>=0.3.2 mmdet mmsegmentation timm diff --git a/setup.cfg b/setup.cfg index 3349b9f7f..f3eceac29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,7 @@ split_before_expression_after_opening_paren = true [isort] line_length = 79 multi_line_output = 0 -known_standard_library = pkg_resources,setuptools +extra_standard_library = pkg_resources,setuptools known_first_party = mmrazor known_third_party=cv2,mmcls,mmcv,mmdet,mmseg,numpy,ordered_set,packaging,pytest,pytorch_sphinx_theme,torch,yaml no_lines_before = STDLIB,LOCALFOLDER diff --git a/setup.py b/setup.py index ca1c96546..853cbe985 100644 --- a/setup.py +++ b/setup.py @@ -163,10 +163,7 @@ def add_mim_extension(): description='OpenMMLab Model Compression Toolbox and Benchmark', long_description=readme(), long_description_content_type='text/markdown', - author='MMRazor Contributors', - author_email='openmmlab@gmail.com', keywords='computer vision, model compression', - url='https://github.com/open-mmlab/mmrazor', packages=find_packages(exclude=('configs', 'tools', 'demo')), include_package_data=True, classifiers=[ @@ -179,8 +176,16 @@ def add_mim_extension(): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', ], + url='https://github.com/open-mmlab/mmrazor', + author='MMRazor Contributors', + author_email='openmmlab@gmail.com', license='Apache License 2.0', - tests_require=parse_requirements('requirements/tests.txt'), install_requires=parse_requirements('requirements/runtime.txt'), + extras_require={ + 'all': parse_requirements('requirements.txt'), + 'tests': parse_requirements('requirements/tests.txt'), + 'optional': parse_requirements('requirements/optional.txt'), + }, zip_safe=False) diff --git a/tests/data/cwd_pspnet.py b/tests/data/cwd_pspnet.py index 376e244b7..ca609897f 100644 --- a/tests/data/cwd_pspnet.py +++ b/tests/data/cwd_pspnet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. norm_cfg = dict(type='BN',requires_grad=True) diff --git a/tests/data/detnas_frcnn_shufflenet_fpn.py b/tests/data/detnas_frcnn_shufflenet_fpn.py index 0720aadb1..35e909058 100644 --- a/tests/data/detnas_frcnn_shufflenet_fpn.py +++ b/tests/data/detnas_frcnn_shufflenet_fpn.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. norm_cfg = dict(type='SyncBN', requires_grad=True) model = dict( type='mmdet.FasterRCNN', diff --git a/tests/data/retinanet.py b/tests/data/retinanet.py index 478163373..fb05e1947 100644 --- a/tests/data/retinanet.py +++ b/tests/data/retinanet.py @@ -1,3 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. # small RetinaNet num_classes=3 diff --git a/tests/test_models/test_algorithms/test_algorithm.py b/tests/test_models/test_algorithms/test_algorithm.py index d7fc93891..4ec729387 100644 --- a/tests/test_models/test_algorithms/test_algorithm.py +++ b/tests/test_models/test_algorithms/test_algorithm.py @@ -236,6 +236,90 @@ def test_spos(): assert flops_supernet > flops_subnet_spos > 0 +def test_spos_mobilenet(): + + model_cfg = dict( + type='mmcls.ImageClassifier', + backbone=dict(type='SearchableMobileNet', widen_factor=1.0), + neck=dict(type='mmcls.GlobalAveragePooling'), + head=dict( + type='mmcls.LinearClsHead', + num_classes=1000, + in_channels=1280, + loss=dict(type='mmcls.CrossEntropyLoss', loss_weight=1.0), + topk=(1, 5), + ), + ) + + architecture_cfg = dict( + type='MMClsArchitecture', + model=model_cfg, + ) + + mutator_cfg = dict( + type='OneShotMutator', + placeholder_mapping=dict( + searchable_blocks=dict( + type='OneShotOP', + choices=dict( + mb_k3e3=dict( + type='MBBlock', + kernel_size=3, + expand_ratio=3, + act_cfg=dict(type='ReLU6')), + mb_k5e3=dict( + type='MBBlock', + kernel_size=5, + expand_ratio=3, + act_cfg=dict(type='ReLU6')), + mb_k7e3=dict( + type='MBBlock', + kernel_size=7, + expand_ratio=3, + act_cfg=dict(type='ReLU6')), + mb_k3e6=dict( + type='MBBlock', + kernel_size=3, + expand_ratio=6, + act_cfg=dict(type='ReLU6')), + mb_k5e6=dict( + type='MBBlock', + kernel_size=5, + expand_ratio=6, + act_cfg=dict(type='ReLU6')), + mb_k7e6=dict( + type='MBBlock', + kernel_size=7, + expand_ratio=6, + act_cfg=dict(type='ReLU6')), + identity=dict(type='Identity'))), + first_blocks=dict( + type='OneShotOP', + choices=dict( + mb_k3e1=dict( + type='MBBlock', + kernel_size=3, + expand_ratio=1, + act_cfg=dict(type='ReLU6')), )))) + + algorithm_cfg = dict( + type='SPOS', + architecture=architecture_cfg, + mutator=mutator_cfg, + retraining=False, + ) + + imgs = torch.randn(16, 3, 224, 224) + label = torch.randint(0, 1000, (16, )) + + algorithm_cfg_ = deepcopy(algorithm_cfg) + algorithm = ALGORITHMS.build(algorithm_cfg_) + + # test forward + losses = algorithm(imgs, return_loss=True, gt_label=label) + assert losses['loss'].item() > 0 + + def test_detnas(): config_path = os.path.join( dirname(dirname(dirname(__file__))), diff --git a/tests/test_models/test_op.py b/tests/test_models/test_op.py index 8810e23dc..4b4fdf40c 100644 --- a/tests/test_models/test_op.py +++ b/tests/test_models/test_op.py @@ -4,6 +4,40 @@ from mmrazor.models.builder import OPS +def test_common_ops(): + tensor = torch.randn(16, 16, 32, 32) + + # test stride != 1 + identity_cfg = dict( + type='Identity', in_channels=16, out_channels=16, stride=2) + + op = OPS.build(identity_cfg) + + # test forward + outputs = op(tensor) + assert outputs.size(1) == 16 and outputs.size(2) == 16 + + # test stride == 1 + identity_cfg = dict( + type='Identity', in_channels=16, out_channels=16, stride=1) + + op = OPS.build(identity_cfg) + + # test forward + outputs = op(tensor) + assert outputs.size(1) == 16 and outputs.size(2) == 32 + + # test in_channels != out_channels + identity_cfg = dict( + type='Identity', in_channels=8, out_channels=16, stride=1) + + op = OPS.build(identity_cfg) + + # test forward + outputs = op(tensor[:, :8]) + assert outputs.size(1) == 16 and outputs.size(2) == 32 + + def test_shuffle_series(): tensor = torch.randn(16, 16, 32, 32) @@ -61,6 +95,51 @@ def test_shuffle_series(): assert outputs.size(1) == 16 and outputs.size(2) == 32 +def test_mobilenet_series(): + + tensor = torch.randn(16, 16, 32, 32) + + kernel_sizes = (3, 5, 7) + expand_ratios = (3, 6) + strides = (1, 2) + se_cfg_1 = dict( + ratio=4, + act_cfg=(dict(type='HSwish'), + dict( + type='HSigmoid', + bias=3, + divisor=6, + min_value=0, + max_value=1))) + se_cfgs = (None, se_cfg_1) + drop_path_rates = (0, 0.2) + with_cps = (True, False) + + for kernel_size in kernel_sizes: + for expand_ratio in expand_ratios: + for stride in strides: + for se_cfg in se_cfgs: + for drop_path_rate in drop_path_rates: + for with_cp in with_cps: + op_cfg = dict( + type='MBBlock', + in_channels=16, + out_channels=16, + kernel_size=kernel_size, + expand_ratio=expand_ratio, + se_cfg=se_cfg, + drop_path_rate=drop_path_rate, + with_cp=with_cp, + stride=stride) + + op = OPS.build(op_cfg) + + # test forward + outputs = op(tensor) + assert outputs.size(1) == 16 and outputs.size( + 2) == 32 // stride + + def test_darts_series(): tensor = torch.randn(16, 16, 32, 32) diff --git a/tools/mmcls/dist_search.sh b/tools/mmcls/dist_search.sh new file mode 100644 index 000000000..c45483919 --- /dev/null +++ b/tools/mmcls/dist_search.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CONFIG=$1 +CHECKPOINT=$2 +GPUS=$3 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/search_mmcls.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4} diff --git a/tools/mmcls/dist_test.sh b/tools/mmcls/dist_test.sh new file mode 100644 index 000000000..9749e54d7 --- /dev/null +++ b/tools/mmcls/dist_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CONFIG=$1 +CHECKPOINT=$2 +GPUS=$3 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/test_mmcls.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4} diff --git a/tools/mmcls/dist_train.sh b/tools/mmcls/dist_train.sh new file mode 100644 index 000000000..0b6fadcf9 --- /dev/null +++ b/tools/mmcls/dist_train.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CONFIG=$1 +GPUS=$2 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/train_mmcls.py $CONFIG --launcher pytorch ${@:3} diff --git a/tools/mmcls/slurm_search_mmcls.sh b/tools/mmcls/slurm_search_mmcls.sh index b9beb75d0..81f7c5a02 100644 --- a/tools/mmcls/slurm_search_mmcls.sh +++ b/tools/mmcls/slurm_search_mmcls.sh @@ -11,7 +11,7 @@ GPUS=${GPUS:-4} GPUS_PER_NODE=${GPUS_PER_NODE:-8} CPUS_PER_TASK=${CPUS_PER_TASK:-12} SRUN_ARGS=${SRUN_ARGS:-""} -PY_ARGS=${@:5} +PY_ARGS=${@:6} PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ srun -p ${PARTITION} \ diff --git a/tools/mmdet/dist_search.sh b/tools/mmdet/dist_search.sh new file mode 100644 index 000000000..510447c02 --- /dev/null +++ b/tools/mmdet/dist_search.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CONFIG=$1 +CHECKPOINT=$2 +GPUS=$3 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/search_mmdet.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4} diff --git a/tools/mmdet/dist_test.sh b/tools/mmdet/dist_test.sh new file mode 100644 index 000000000..e710ba9fc --- /dev/null +++ b/tools/mmdet/dist_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CONFIG=$1 +CHECKPOINT=$2 +GPUS=$3 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/test_mmdet.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4} diff --git a/tools/mmdet/dist_train.sh b/tools/mmdet/dist_train.sh new file mode 100644 index 000000000..16d3167f4 --- /dev/null +++ b/tools/mmdet/dist_train.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CONFIG=$1 +GPUS=$2 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/train_mmdet.py $CONFIG --launcher pytorch ${@:3} diff --git a/tools/mmseg/dist_test.sh b/tools/mmseg/dist_test.sh new file mode 100644 index 000000000..2e1d6ddd0 --- /dev/null +++ b/tools/mmseg/dist_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CONFIG=$1 +CHECKPOINT=$2 +GPUS=$3 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/test_mmseg.py $CONFIG $CHECKPOINT --launcher pytorch ${@:4} diff --git a/tools/mmseg/dist_train.sh b/tools/mmseg/dist_train.sh new file mode 100644 index 000000000..c38ef173e --- /dev/null +++ b/tools/mmseg/dist_train.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CONFIG=$1 +GPUS=$2 +PORT=${PORT:-29500} + +PYTHONPATH="$(dirname $0)/../..":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=$GPUS --master_port=$PORT \ + $(dirname "$0")/train_mmseg.py $CONFIG --launcher pytorch ${@:3}