[Feature] Add MMRazor quantization

.circleci/test.yml

@@ -103,9 +103,9 @@ jobs:
           name: Clone Repos
           command: |
             git clone -b main --depth 1 https://github.com/open-mmlab/mmengine.git /home/circleci/mmengine
-            git clone -b dev-3.x --depth 1 https://github.com/open-mmlab/mmdetection.git /home/circleci/mmdetection
-            git clone -b dev-1.x --depth 1 https://github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
-            git clone -b dev-1.x --depth 1 https://github.com/open-mmlab/mmsegmentation.git /home/circleci/mmsegmentation
+            git clone -b main --depth 1 https://github.com/open-mmlab/mmdetection.git /home/circleci/mmdetection
+            git clone -b 1.x --depth 1 https://github.com/open-mmlab/mmclassification.git /home/circleci/mmclassification
+            git clone -b main --depth 1 https://github.com/open-mmlab/mmsegmentation.git /home/circleci/mmsegmentation
       - run:
           name: Build Docker image
           command: |
@@ -153,15 +153,15 @@ workflows:
                 - dev-1.x
       - build_cpu:
           name: minimum_version_cpu
-          torch: 1.6.0
-          torchvision: 0.7.0
-          python: 3.7.9
+          torch: 1.8.1
+          torchvision: 0.9.1
+          python: 3.7.4
           requires:
             - lint
       - build_cpu:
           name: maximum_version_cpu
-          torch: 1.12.1
-          torchvision: 0.13.1
+          torch: 1.13.1
+          torchvision: 0.14.1
           python: 3.9.0
           requires:
             - lint
@@ -183,7 +183,7 @@ workflows:
     jobs:
       - build_cuda:
           name: minimum_version_gpu
-          torch: 1.6.0
+          torch: 1.8.1
           # Use double quotation mark to explicitly specify its type
           # as string instead of number
           cuda: "10.1"

.dev_scripts/benchmark_summary_analyse.py

@@ -0,0 +1,67 @@
+import argparse
+import os
+
+import mmengine
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Analyse summary.yml generated by benchmark test')
+    parser.add_argument('file_path', help='Summary.yml path')
+    args = parser.parse_args()
+    return args
+
+
+metric_mapping = {
+    'Top 1 Accuracy': 'accuracy/top1',
+    'Top 5 Accuracy': 'accuracy/top5',
+    'box AP': 'coco/bbox_mAP',
+    'mIoU': 'mIoU'
+}
+
+
+def compare_metric(result, metric):
+    expect_val = result['expect'][metric]
+    actual_val = result['actual'].get(metric_mapping[metric], None)
+    if actual_val is None:
+        return None, None
+    if metric == 'box AP':
+        actual_val *= 100
+    decimal_bit = len(str(expect_val).split('.')[-1])
+    actual_val = round(actual_val, decimal_bit)
+    error = round(actual_val - expect_val, decimal_bit)
+    error_percent = round(abs(error) * 100 / expect_val, 3)
+    return error, error_percent
+
+
+def main():
+    args = parse_args()
+    file_path = args.file_path
+    results = mmengine.load(file_path, 'yml')
+    miss_models = dict()
+    sort_by_error = dict()
+    for k, v in results.items():
+        valid_keys = v['expect'].keys()
+        compare_res = dict()
+        for m in valid_keys:
+            error, error_percent = compare_metric(v, m)
+            if error is None:
+                continue
+            compare_res[m] = {'error': error, 'error_percent': error_percent}
+            if error != 0:
+                miss_models[k] = compare_res
+                sort_by_error[k] = error
+    sort_by_error = sorted(
+        sort_by_error.items(), key=lambda x: abs(x[1]), reverse=True)
+    miss_models_sort = dict()
+    miss_models_sort['total error models'] = len(sort_by_error)
+    for k_v in sort_by_error:
+        index = k_v[0]
+        miss_models_sort[index] = miss_models[index]
+    save_path = os.path.join(os.path.dirname(file_path), 'summary_error.yml')
+    mmengine.fileio.dump(miss_models_sort, save_path, sort_keys=False)
+    print(f'Summary analysis result saved in {save_path}')
+
+
+if __name__ == '__main__':
+    main()

.dev_scripts/benchmark_test.py

@@ -24,9 +24,9 @@
 def parse_args():
     parser = argparse.ArgumentParser(
         description="Test all models' accuracy in model-index.yml")
-    parser.add_argument(
-        'partition', type=str, help='Cluster partition to use.')
     parser.add_argument('checkpoint_root', help='Checkpoint file root path.')
+    parser.add_argument(
+        '--partition', type=str, help='Cluster partition to use.')
     parser.add_argument(
         '--job-name',
         type=str,
@@ -148,6 +148,7 @@ def create_test_job_batch(commands, model_info, args, port):
     if exists:
         print(f'{checkpoint} already exists.')
     else:
+        print(f'start downloading {fname}')
         wget.download(model_info.weights, str(checkpoint))
         print(f'\nSaved in {checkpoint}.')
 

.github/workflows/build.yml

@@ -25,22 +25,12 @@ concurrency:
 
 jobs:
   test_linux:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python-version: [3.7]
-        torch: [1.6.0, 1.7.0, 1.8.0, 1.9.0, 1.10.0, 1.11.0, 1.12.0, 1.13.0]
+        torch: [1.8.0, 1.9.0, 1.10.0, 1.11.0, 1.12.0, 1.13.0]
         include:
-          - torch: 1.6.0
-            torch_version: 1.6
-            torchvision: 0.7.0
-          - torch: 1.7.0
-            torch_version: 1.7
-            torchvision: 0.8.1
-          - torch: 1.7.0
-            torch_version: 1.7
-            torchvision: 0.8.1
-            python-version: 3.8
           - torch: 1.8.0
             torch_version: 1.8
             torchvision: 0.9.0
@@ -103,11 +93,11 @@ jobs:
           pip install -U openmim
           mim install 'mmcv >= 2.0.0rc1'
       - name: Install MMCls
-        run: pip install git+https://github.com/open-mmlab/mmclassification.git@dev-1.x
+        run: pip install 'mmcls>=1.0.0rc0'
       - name: Install MMDet
-        run: pip install git+https://github.com/open-mmlab/mmdetection.git@dev-3.x
+        run: pip install git+https://github.com/open-mmlab/mmdetection.git@main
       - name: Install MMSeg
-        run: pip install git+https://github.com/open-mmlab/mmsegmentation.git@dev-1.x
+        run: pip install git+https://github.com/open-mmlab/mmsegmentation.git@main
       - name: Install other dependencies
         run: pip install -r requirements.txt
       - name: Build and install
@@ -119,8 +109,8 @@ jobs:
           coverage report -m
       # Upload coverage report for python3.8 && pytorch1.12.0 cpu
       - name: Upload coverage to Codecov
-        if: ${{matrix.torch == '1.12.0' && matrix.python-version == '3.8'}}
-        uses: codecov/codecov-action@v2
+        if: ${{matrix.torch == '1.13.0' && matrix.python-version == '3.8'}}
+        uses: codecov/codecov-action@v3
         with:
           file: ./coverage.xml
           flags: unittests

README.md

@@ -21,7 +21,7 @@
 <!--算法库 Badges-->
 
 [![PyPI](https://img.shields.io/pypi/v/mmrazor)](https://pypi.org/project/mmrazor)
-[![docs](https://img.shields.io/badge/docs-latest-blue)](https://mmrazor.readthedocs.io/en/dev-1.x/)
+[![docs](https://img.shields.io/badge/docs-latest-blue)](https://mmrazor.readthedocs.io/en/quantize/)
 [![badge](https://github.com/open-mmlab/mmrazor/workflows/build/badge.svg)](https://github.com/open-mmlab/mmrazor/actions)
 [![codecov](https://codecov.io/gh/open-mmlab/mmrazor/branch/master/graph/badge.svg)](https://codecov.io/gh/open-mmlab/mmrazor)
 [![license](https://img.shields.io/github/license/open-mmlab/mmrazor.svg)](https://github.com/open-mmlab/mmrazor/blob/master/LICENSE)
@@ -32,9 +32,9 @@
 
 <!--Note:请根据各算法库自身情况设置项目和链接-->
 
-[📘Documentation](https://mmrazor.readthedocs.io/en/dev-1.x/) |
-[🛠️Installation](https://mmrazor.readthedocs.io/en/dev-1.x/get_started/installation.html) |
-[👀Model Zoo](https://mmrazor.readthedocs.io/en/dev-1.x/get_started/model_zoo.html) |
+[📘Documentation](https://mmrazor.readthedocs.io/en/quantize/) |
+[🛠️Installation](https://mmrazor.readthedocs.io/en/quantize/get_started/installation.html) |
+[👀Model Zoo](https://mmrazor.readthedocs.io/en/quantize/get_started/model_zoo.html) |
 [🤔Reporting Issues](https://github.com/open-mmlab/mmrazor/issues/new/choose)
 
 </div>
@@ -68,7 +68,7 @@ MMRazor is a model compression toolkit for model slimming and AutoML, which incl
 - Neural Architecture Search (NAS)
 - Pruning
 - Knowledge Distillation (KD)
-- Quantization (come soon)
+- Quantization
 
 It is a part of the [OpenMMLab](https://openmmlab.com/) project.
 
@@ -86,7 +86,7 @@ Major features:
 
   With better modular design, developers can implement new model compression algorithms with only a few codes, or even by simply modifying config files.
 
-Below is an overview of MMRazor's design and implementation, please refer to [tutorials](https://mmrazor.readthedocs.io/en/dev-1.x/get_started/overview.html) for more details.
+Below is an overview of MMRazor's design and implementation, please refer to [tutorials](https://mmrazor.readthedocs.io/en/quantize/get_started/overview.html) for more details.
 
 <div align="center">
   <img src="resources/design_and_implement.png" style="zoom:100%"/>
@@ -164,7 +164,7 @@ Please refer to [installation.md](/docs/en/get_started/installation.md) for more
 
 ## Getting Started
 
-Please refer to [user guides](https://mmrazor.readthedocs.io/en/dev-1.x/user_guides/index.html) for the basic usage of MMRazor. There are also [advanced guides](https://mmrazor.readthedocs.io/en/dev-1.x/advanced_guides/index.html):
+Please refer to [user guides](https://mmrazor.readthedocs.io/en/quantize/user_guides/index.html) for the basic usage of MMRazor. There are also [advanced guides](https://mmrazor.readthedocs.io/en/quantize/advanced_guides/index.html):
 
 ## Contributing
 

configs/distill/mmcls/ofd/README.md

@@ -22,16 +22,16 @@ We investigate the design aspects of feature distillation methods achieving netw
 
 #### Vanilla
 
-| Dataset | Model                                                                   | Top-1 (%) | Top-5 (%) | Download                                                                                                                                                                                                                              |
-| ------- | ----------------------------------------------------------------------- | --------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| CIFAR10 | [WRN16-2](../../../vanilla/mmcls/wide-resnet/wrn16-w2_b16x8_cifar10.py) | 93.43     | 99.75     | [model](https://download.openmmlab.com/mmrazor/v1/wide_resnet/wrn16_2_b16x8_cifar10_20220831_204709-446b466e.pth) \| [log](https://download.openmmlab.com/mmrazor/v1/wide_resnet/wrn16_2_b16x8_cifar10_20220831_204709-446b466e.json) |
-| CIFAR10 | [WRN28-4](../../../vanilla/mmcls/wide-resnet/wrn28-w4_b16x8_cifar10.py) | 95.49     | 99.81     | [model](https://download.openmmlab.com/mmrazor/v1/wide_resnet/wrn28_4_b16x8_cifar10_20220831_173536-d6f8725c.pth) \| [log](https://download.openmmlab.com/mmrazor/v1/wide_resnet/wrn28_4_b16x8_cifar10_20220831_173536-d6f8725c.json) |
+| Dataset | Model                                                                   | Top-1 (%) | Download                                                                                                                                                                                                                              |
+| ------- | ----------------------------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| CIFAR10 | [WRN16-2](../../../vanilla/mmcls/wide-resnet/wrn16-w2_b16x8_cifar10.py) | 93.43     | [model](https://download.openmmlab.com/mmrazor/v1/wide_resnet/wrn16_2_b16x8_cifar10_20220831_204709-446b466e.pth) \| [log](https://download.openmmlab.com/mmrazor/v1/wide_resnet/wrn16_2_b16x8_cifar10_20220831_204709-446b466e.json) |
+| CIFAR10 | [WRN28-4](../../../vanilla/mmcls/wide-resnet/wrn28-w4_b16x8_cifar10.py) | 95.49     | [model](https://download.openmmlab.com/mmrazor/v1/wide_resnet/wrn28_4_b16x8_cifar10_20220831_173536-d6f8725c.pth) \| [log](https://download.openmmlab.com/mmrazor/v1/wide_resnet/wrn28_4_b16x8_cifar10_20220831_173536-d6f8725c.json) |
 
 #### Distillation
 
-| Dataset | Model   | Flops(M) | Teacher | Top-1 (%) | Top-5 (%) | Configs                                                     | Download                                                                                                                                                                                                                                                                      |
-| ------- | ------- | -------- | ------- | --------- | --------- | ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| CIFAR10 | WRN16-2 | 101      | WRN28-4 | 95.23     | 99.79     | [config](./ofd_backbone_resnet50_resnet18_8xb16_cifar10.py) | [model](https://download.openmmlab.com/mmrazor/v1/overhaul/ofd_backbone_resnet50_resnet18_8xb16_cifar10_20220831_220553-f5d12e61.pth) \| [log](https://download.openmmlab.com/mmrazor/v1/overhaul/ofd_backbone_resnet50_resnet18_8xb16_cifar10_20220831_220553-f5d12e61.json) |
+| Dataset | Model   | Flops(M) | Teacher | Top-1 (%) | Configs                                                     | Download                                                                                                                                                                                                                                                                     |
+| ------- | ------- | -------- | ------- | --------- | ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| CIFAR10 | WRN16-2 | 101      | WRN28-4 | 94.21     | [config](./ofd_backbone_resnet50_resnet18_8xb16_cifar10.py) | [model](https://download.openmmlab.com/mmrazor/v1/overhaul/ofd_backbone_resnet50_resnet18_8xb16_cifar10_20230417_192216-ace2908f.pth) \| [log](https://download.openmmlab.com/mmrazor/v1/overhaul/ofd_backbone_resnet50_resnet18_8xb16_cifar10_20230417_192216-ace2908f.log) |
 
 ## Getting Started
 

configs/distill/mmcls/ofd/metafile.yml

@@ -33,6 +33,6 @@ Models:
       - Task: Image Classification
         Dataset: CIFAR-10
         Metrics:
-          Top 1 Accuracy: 95.4400
+          Top 1 Accuracy: 94.21
     Config: configs/distill/mmcls/ofd/ofd_backbone_resnet50_resnet18_8xb16_cifar10.py
-    Weights: https://download.openmmlab.com/mmrazor/v1/overhaul/ofd_backbone_resnet50_resnet18_8xb16_cifar10_20220831_220553-f5d12e61.pth
+    Weights: https://download.openmmlab.com/mmrazor/v1/overhaul/ofd_backbone_resnet50_resnet18_8xb16_cifar10_20230417_192216-ace2908f.pth

configs/nas/mmcls/darts/darts_subnet_1xb96_cifar10_2.0.py

@@ -37,7 +37,7 @@
     init_cfg=dict(
         type='Pretrained',
         checkpoint=  # noqa: E251
-        'https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v1/darts/darts_subnetnet_1xb96_cifar10_acc-97.27_20211222-17e42600_latest.pth',  # noqa: E501
+        'https://download.openmmlab.com/mmrazor/v1/darts/darts_subnetnet_1xb96_cifar10_acc-97.27_20211222-17e42600_latest.pth',  # noqa: E501
         prefix='architecture.'))
 
 model_wrapper_cfg = None

configs/nas/mmcls/darts/metafile.yml

@@ -25,4 +25,4 @@ Models:
           Top 1 Accuracy: 97.32
           Top 5 Accuracy: 99.94
     Config: configs/nas/mmcls/darts/darts_subnet_1xb96_cifar10_2.0.py
-    Weights: https://download.openmmlab.com/mmrazor/v1/darts/darts_subnetnet_1xb96_cifar10_acc-97.32_20211222-23ca1e10.pth
+    Weights: https://download.openmmlab.com/mmrazor/v1/darts/darts_subnetnet_1xb96_cifar10_acc-97.32_20211222-e5727921_latest.pth

configs/nas/mmdet/detnas/detnas_frcnn_shufflenet_subnet_coco_1x.py

@@ -9,7 +9,7 @@
     init_cfg=dict(
         type='Pretrained',
         checkpoint=  # noqa: E251
-        'detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20220715-61d2e900_v1.pth',  # noqa: E501
+        'https://download.openmmlab.com/mmrazor/v1/detnas/detnas_subnet_frcnn_shufflenetv2_fpn_1x_coco_bbox_backbone_flops-0.34M_mAP-37.5_20220715-61d2e900_v1.pth',  # noqa: E501
         prefix='architecture.'))
 
 find_unused_parameters = False