diff --git a/.pylintrc b/.pylintrc index 4ba02ef..a30c7ac 100644 --- a/.pylintrc +++ b/.pylintrc @@ -558,4 +558,4 @@ valid-metaclass-classmethod-first-arg=mcs # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=base.Exception diff --git a/AcceptanceTests/CMakeLists.txt b/AcceptanceTests/CMakeLists.txt index 09515f0..198f07a 100644 --- a/AcceptanceTests/CMakeLists.txt +++ b/AcceptanceTests/CMakeLists.txt @@ -15,16 +15,18 @@ if (ENABLE_TORCH) endif (ENABLE_TORCH) -#if (ENABLE_TENSORFLOW) -# add_dependencies(AcceptanceTests -# AcceptanceTests.Tensorflow) -# -#endif (ENABLE_TENSORFLOW) +if (ENABLE_TENSORFLOW) + add_dependencies(AcceptanceTests + AcceptanceTests.Tensorflow) + +endif (ENABLE_TENSORFLOW) if (ENABLE_TORCH) + message(STATUS "Torch has been enabled") add_subdirectory(torch) endif (ENABLE_TORCH) -#if (ENABLE_TENSORFLOW) -# add_subdirectory(tensorflow) -#endif (ENABLE_TENSORFLOW) +if (ENABLE_TENSORFLOW) + message(STATUS "Tensorflow has been enabled") + add_subdirectory(tensorflow) +endif (ENABLE_TENSORFLOW) diff --git a/AcceptanceTests/tensorflow/CMakeLists.txt b/AcceptanceTests/tensorflow/CMakeLists.txt new file mode 100644 index 0000000..902ed88 --- /dev/null +++ b/AcceptanceTests/tensorflow/CMakeLists.txt @@ -0,0 +1,32 @@ +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +if (ENABLE_CUDA) + set(CUDA_FLAG -m "not blah") + set(USE_CUDA True) +else (ENABLE_CUDA) + set(CUDA_FLAG -m "not cuda") + set(USE_CUDA False) +endif (ENABLE_CUDA) + +add_custom_target( AcceptanceTests.Tensorflow ) + +file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/*quanteval.py") + foreach(filename ${files}) + get_filename_component( testname "${filename}" NAME_WE ) + string(SUBSTRING ${testname} 5 -1 formatted_name) + message(STATUS "Now running Tensorflow Test: ${formatted_name}") + + add_custom_target(AcceptanceTests.Tensorflow.${testname} + VERBATIM COMMAND . /opt/${formatted_name}-venv/bin/activate && export HF_HOME=/opt/cache && + ${CMAKE_COMMAND} -E env + ${MZ_PYTHONPATH}:$$PYTHONPATH + python -m pytest -s ${filename} + ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml) + + add_dependencies( AcceptanceTests.Tensorflow AcceptanceTests.Tensorflow.${testname} ) + endforeach( filename ) \ No newline at end of file diff --git a/AcceptanceTests/tensorflow/conftest.py b/AcceptanceTests/tensorflow/conftest.py old mode 100644 new mode 100755 index ba8769c..28b1446 --- a/AcceptanceTests/tensorflow/conftest.py +++ b/AcceptanceTests/tensorflow/conftest.py @@ -52,4 +52,20 @@ def tiny_mscoco_tfrecords(test_data_path): yield tiny_mscoco_tfrecords +@pytest.fixture(scope='module') +def PascalVOC_segmentation_test_data_path(test_data_path): + if test_data_path is not None: + pascalVOC_segmentation_path = (test_data_path / "model_zoo_datasets/tiny_VOCdevkit").as_posix() + else: + pascalVOC_segmentation_path = None + + yield pascalVOC_segmentation_path + +#@pytest.fixture(scope='module') +#def tiny_imageNet_tfrecords(test_data_path): +# if test_data_path is not None: +# tiny_imageNet_tfrecords = (test_data_path / "model_zoo_datasets/tiny_imageNet_tfrecords").as_posix() +# else: +# tiny_imageNet_tfrecords = None +# yield tiny_imageNet_tfrecords diff --git a/AcceptanceTests/tensorflow/staging/test_deeplabv3plus_mbnv2_tf2_quanteval.py b/AcceptanceTests/tensorflow/staging/test_deeplabv3plus_mbnv2_tf2_quanteval.py new file mode 100644 index 0000000..a42f41f --- /dev/null +++ b/AcceptanceTests/tensorflow/staging/test_deeplabv3plus_mbnv2_tf2_quanteval.py @@ -0,0 +1,37 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for deeplabv3plus mbnv2""" + +import pytest +from aimet_zoo_tensorflow.deeplabv3plus_tf2.evaluators import deeplabv3plus_tf2_quanteval + +@pytest.mark.slow +@pytest.mark.cuda +@pytest.mark.sementic_segmentation +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["deeplabv3plus_mbnv2_w8a8"]) +def test_quanteval_deeplabv3plus_mbnv2_tf2(model_config, PascalVOC_segmentation_test_data_path): + """mobiledet edgetpu image classification test""" + + if PascalVOC_segmentation_test_data_path is None: + pytest.xfail(f'Dataset path is not set') + + deeplabv3plus_tf2_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + PascalVOC_segmentation_test_data_path + ] + ) + diff --git a/AcceptanceTests/tensorflow/staging/test_deeplabv3plus_xception_tf2_quanteval.py b/AcceptanceTests/tensorflow/staging/test_deeplabv3plus_xception_tf2_quanteval.py new file mode 100644 index 0000000..03b0cb0 --- /dev/null +++ b/AcceptanceTests/tensorflow/staging/test_deeplabv3plus_xception_tf2_quanteval.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for deeplabv3plus xception""" + +import pytest +from aimet_zoo_tensorflow.deeplabv3plus_tf2.evaluators import deeplabv3plus_tf2_quanteval + +@pytest.mark.slow +@pytest.mark.cuda +@pytest.mark.sementic_segmentation +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["deeplabv3plus_xception_w8a8"]) +def test_quanteval_deeplabv3plus_xception_tf2(model_config, PascalVOC_segmentation_test_data_path): + """mobiledet edgetpu image classification test""" + + if PascalVOC_segmentation_test_data_path is None: + pytest.xfail(f'Dataset path is not set') + + deeplabv3plus_tf2_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + PascalVOC_segmentation_test_data_path + ] + ) + diff --git a/AcceptanceTests/tensorflow/staging/test_mobilenet_edgetpu_quanteval.py b/AcceptanceTests/tensorflow/staging/test_mobilenet_edgetpu_quanteval.py new file mode 100755 index 0000000..b19fda1 --- /dev/null +++ b/AcceptanceTests/tensorflow/staging/test_mobilenet_edgetpu_quanteval.py @@ -0,0 +1,34 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for resnet""" +import pytest +from aimet_zoo_tensorflow.mobilenetedgetpu.evaluators import mobilenet_edgetpu_quanteval + +@pytest.mark.cuda +@pytest.mark.image_classification +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["mobilenetedgetpu_w8a8"]) +def test_quanteval_mobilenet_edgetpu(model_config, tiny_imageNet_tfrecords): + """resnet image classification test""" + + if tiny_imageNet_tfrecords is None: + pytest.xfail(f'failed since dataset path is not set') + + mobilenet_edgetpu_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_imageNet_tfrecords, + ] + ) \ No newline at end of file diff --git a/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py b/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py index 46a2be0..5a8f14e 100644 --- a/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py +++ b/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py @@ -18,7 +18,7 @@ @pytest.mark.cuda @pytest.mark.object_detection # pylint:disable = redefined-outer-name -@pytest.mark.parametrize("model_config", ["resnet50_w8a8"]) +@pytest.mark.parametrize("model_config", ["mobilenetv2_w8a8"]) def test_quanteval_mobilenet_v2(model_config, tiny_imageNet_root_path): """mobilenet_v2_tf2 image classification acceptance test""" diff --git a/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py b/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py index ffa21bc..d07e468 100644 --- a/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py +++ b/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py @@ -13,7 +13,7 @@ """ acceptance test for ssd_mobilenetv2_quanteval edgetpu""" import pytest -from aimet_zoo_tensorflow.ssd_mobilenet_v2.evaluators import ssd_mobilenetv2_quanteval +from aimet_zoo_tensorflow.ssd_mobilenet_v2.evaluators import ssd_mobilenet_v2_quanteval @pytest.mark.slow @pytest.mark.cuda @@ -26,7 +26,7 @@ def test_quanteval_ssd_mobilenetv2(model_config, tiny_mscoco_tfrecords): if tiny_mscoco_tfrecords is None: pytest.fail(f'Dataset path is not set') - ssd_mobilenetv2_quanteval.main( + ssd_mobilenet_v2_quanteval.main( [ "--model-config", model_config, diff --git a/AcceptanceTests/torch/CMakeLists.txt b/AcceptanceTests/torch/CMakeLists.txt index f3f9685..2d586dc 100644 --- a/AcceptanceTests/torch/CMakeLists.txt +++ b/AcceptanceTests/torch/CMakeLists.txt @@ -15,16 +15,17 @@ endif (ENABLE_CUDA) add_custom_target( AcceptanceTests.Torch ) -file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/test_resnet_quanteval.py") +file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/*quanteval.py") foreach(filename ${files}) - get_filename_component( testname "${filename}" NAME_WE ) - message(STATUS "Testname: " testname) - + string(SUBSTRING ${testname} 5 -1 formatted_name) + message(STATUS "Now running Pytorch Test: ${formatted_name}") + add_custom_target(AcceptanceTests.Torch.${testname} - VERBATIM COMMAND ${CMAKE_COMMAND} -E env - ${MZ_PYTHONPATH} - ${Python3_EXECUTABLE} -m pytest -s ${filename} + VERBATIM COMMAND . /opt/${formatted_name}-venv/bin/activate && export HF_HOME=/opt/cache && + ${CMAKE_COMMAND} -E env + ${MZ_PYTHONPATH}:$$PYTHONPATH + python -m pytest -s ${filename} ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml) add_dependencies( AcceptanceTests.Torch AcceptanceTests.Torch.${testname} ) diff --git a/AcceptanceTests/torch/conftest.py b/AcceptanceTests/torch/conftest.py index 0518640..3ac2390 100644 --- a/AcceptanceTests/torch/conftest.py +++ b/AcceptanceTests/torch/conftest.py @@ -69,7 +69,7 @@ def tiny_mscoco_validation_path(test_data_path): else: tiny_mscoco_validation_path = None - yield tiny_mscoco_validation_path.as_posix() + yield tiny_mscoco_validation_path @pytest.fixture(scope='module') diff --git a/AcceptanceTests/torch/pytest.ini b/AcceptanceTests/torch/pytest.ini index ba52e67..2ff5d2e 100644 --- a/AcceptanceTests/torch/pytest.ini +++ b/AcceptanceTests/torch/pytest.ini @@ -15,5 +15,5 @@ markers = nlp: tests that belong to natual language process task object_detection: tests that belong to object detection task pose_estimation: tests that belong to pose estimation task - sementic_segmentation: tests that belong to sementic segmentation task + semantic_segmentation: tests that belong to sementic segmentation task super_resolution: tests that belong to super resolution task diff --git a/AcceptanceTests/torch/staging/test_ffnet_quanteval.py b/AcceptanceTests/torch/staging/test_ffnet_quanteval.py new file mode 100644 index 0000000..f960de4 --- /dev/null +++ b/AcceptanceTests/torch/staging/test_ffnet_quanteval.py @@ -0,0 +1,65 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" acceptance test for ffnet semantic segmentation""" +import pytest +from math import isclose + +import torch + +from aimet_zoo_torch.ffnet.evaluators import ( + ffnet_quanteval, +) + +expected_results = { + 'segmentation_ffnet40S_dBBB_mobile': {'original_mIoU': 0.7015, 'quantized_mIoU': 0.7018}, + 'segmentation_ffnet54S_dBBB_mobile': {'original_mIoU': 0.6957, 'quantized_mIoU': 0.7368}, + 'segmentation_ffnet78S_BCC_mobile_pre_down': {'original_mIoU': None, 'quantized_mIoU': None}, + 'segmentation_ffnet78S_dBBB_mobile': {'original_mIoU': 0.6904, 'quantized_mIoU': 0.6882}, + 'segmentation_ffnet122NS_CCC_mobile_pre_down': {'original_mIoU': None, 'quantized_mIoU': None} +} + +@pytest.mark.sementic_segmentation +@pytest.mark.cuda +#pylint:disable = redefined-outer-name +@pytest.mark.parametrize( + "model_config, expected_mIoUs",[ + ("segmentation_ffnet40S_dBBB_mobile", expected_results["segmentation_ffnet40S_dBBB_mobile"]), + ("segmentation_ffnet54S_dBBB_mobile", expected_results["segmentation_ffnet54S_dBBB_mobile"]), + ("segmentation_ffnet78S_BCC_mobile_pre_down", expected_results["segmentation_ffnet78S_BCC_mobile_pre_down"]), + ("segmentation_ffnet78S_dBBB_mobile", expected_results["segmentation_ffnet78S_dBBB_mobile"]), + ("segmentation_ffnet122NS_CCC_mobile_pre_down", expected_results["segmentation_ffnet122NS_CCC_mobile_pre_down"]) + ] + ) +def test_quaneval_ffnet( + model_config, + expected_mIoUs, + tiny_cityscapes_path +): + """acceptance test of hrnet for semantic segmentation""" + torch.cuda.empty_cache() + if tiny_cityscapes_path is None: + pytest.xfail('Dataset is not set') + + #TODO: Fix the two failing model cards + if expected_mIoUs['original_mIoU'] is None: + pytest.skip(f'{model_config} hasn`t passed manual testing!') + + mIoUs = ffnet_quanteval.main( + [ + "--model-config", model_config, + "--dataset-path", tiny_cityscapes_path, + "--batch-size", '2' + ] + ) + + assert isclose(mIoUs['mIoU_orig_fp32'], expected_mIoUs['original_mIoU'], rel_tol=1e-4) + assert isclose(mIoUs['mIoU_optim_int8'], expected_mIoUs['quantized_mIoU'], rel_tol=1e-4) diff --git a/AcceptanceTests/torch/test_hrnet_image_classification_quanteval.py b/AcceptanceTests/torch/staging/test_hrnet_image_classification_quanteval.py similarity index 100% rename from AcceptanceTests/torch/test_hrnet_image_classification_quanteval.py rename to AcceptanceTests/torch/staging/test_hrnet_image_classification_quanteval.py diff --git a/AcceptanceTests/torch/test_hrnet_posenet_quanteval.py b/AcceptanceTests/torch/staging/test_hrnet_posenet_quanteval.py similarity index 92% rename from AcceptanceTests/torch/test_hrnet_posenet_quanteval.py rename to AcceptanceTests/torch/staging/test_hrnet_posenet_quanteval.py index d820221..96a9f4d 100644 --- a/AcceptanceTests/torch/test_hrnet_posenet_quanteval.py +++ b/AcceptanceTests/torch/staging/test_hrnet_posenet_quanteval.py @@ -18,7 +18,7 @@ @pytest.mark.pose_estimation @pytest.mark.cuda -@pytest.parametrize("model_config",["hrnet_posenet_w4a8","hrnet_posenet_w8a8"]) +@pytest.mark.parametrize("model_config",["hrnet_posenet_w4a8","hrnet_posenet_w8a8"]) def test_quaneval_hrnet_posenet(model_config, tiny_mscoco_validation_path): """hrnet_posenet pose estimation test""" torch.cuda.empty_cache() diff --git a/AcceptanceTests/torch/test_hrnet_sem_seg_quanteval.py b/AcceptanceTests/torch/staging/test_hrnet_sem_seg_quanteval.py similarity index 94% rename from AcceptanceTests/torch/test_hrnet_sem_seg_quanteval.py rename to AcceptanceTests/torch/staging/test_hrnet_sem_seg_quanteval.py index 3ba7950..b506b0b 100644 --- a/AcceptanceTests/torch/test_hrnet_sem_seg_quanteval.py +++ b/AcceptanceTests/torch/staging/test_hrnet_sem_seg_quanteval.py @@ -23,7 +23,7 @@ @pytest.mark.sementic_segmentation @pytest.mark.cuda #pylint:disable = redefined-outer-name -@pytest.mark.parametrize("model_config",["hrnet_sem_seg_w4a8","hrnet_sem_seg_w4a8"]) +@pytest.mark.parametrize("model_config",["hrnet_sem_seg_w4a8"]) def test_quaneval_hrnet_sem_seg(model_config, tiny_cityscapes_path, monkeypatch): """acceptance test of hrnet for semantic segmentation""" torch.cuda.empty_cache() diff --git a/AcceptanceTests/torch/staging/test_inverseform_quanteval.py b/AcceptanceTests/torch/staging/test_inverseform_quanteval.py new file mode 100644 index 0000000..bce9510 --- /dev/null +++ b/AcceptanceTests/torch/staging/test_inverseform_quanteval.py @@ -0,0 +1,41 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" acceptance test for inverseform """ +import pytest +from math import isclose + +from aimet_zoo_torch.inverseform.evaluators.inverseform_quanteval import main + +expected_results = { + 'hrnet_16_slim_if': {'original_mIoU': 0.6883, 'quantized_mIoU': 0.6850}, + 'ocrnet_48_if': {'original_mIoU': 0.8499, 'quantized_mIoU': 0.8465} +} + + +@pytest.mark.semantic_segmentation +@pytest.mark.parametrize("model_config, expected_mIoUs", [('hrnet_16_slim_if', expected_results['hrnet_16_slim_if']), + ('ocrnet_48_if', expected_results['ocrnet_48_if'])]) +def test_inverseform_quanteval( + model_config, + expected_mIoUs, + tiny_cityscapes_path +): + if tiny_cityscapes_path is None: + pytest.xfail(f"dataset path is None!") + + args = ['--model-config', model_config, + '--dataset-path', tiny_cityscapes_path, + '--batch-size', '2'] + mIoUs = main(args) + + assert isclose(mIoUs['original_mIoU'], expected_mIoUs['original_mIoU'] ,rel_tol=1e-4) + assert isclose(mIoUs['quantized_mIoU'], expected_mIoUs['quantized_mIoU'], rel_tol=1e-4) diff --git a/AcceptanceTests/torch/test_minilm_quanteval.py b/AcceptanceTests/torch/staging/test_minilm_quanteval.py similarity index 100% rename from AcceptanceTests/torch/test_minilm_quanteval.py rename to AcceptanceTests/torch/staging/test_minilm_quanteval.py diff --git a/AcceptanceTests/torch/test_mobilebert_quanteval.py b/AcceptanceTests/torch/staging/test_mobilebert_quanteval.py similarity index 100% rename from AcceptanceTests/torch/test_mobilebert_quanteval.py rename to AcceptanceTests/torch/staging/test_mobilebert_quanteval.py diff --git a/AcceptanceTests/torch/test_resnext_quanteval.py b/AcceptanceTests/torch/staging/test_resnext_quanteval.py similarity index 96% rename from AcceptanceTests/torch/test_resnext_quanteval.py rename to AcceptanceTests/torch/staging/test_resnext_quanteval.py index ac9ba52..1495ad7 100644 --- a/AcceptanceTests/torch/test_resnext_quanteval.py +++ b/AcceptanceTests/torch/staging/test_resnext_quanteval.py @@ -30,7 +30,7 @@ def test_quanteval_resnext(model_config, tiny_imageNet_validation_path): resnext_quanteval.main( [ "--model-config", - model_config["resnext"], + model_config, "--dataset-path", tiny_imageNet_validation_path, ] diff --git a/AcceptanceTests/torch/test_ssd_mobilenetv2_quanteval.py b/AcceptanceTests/torch/staging/test_ssd_mobilenetv2_quanteval.py similarity index 100% rename from AcceptanceTests/torch/test_ssd_mobilenetv2_quanteval.py rename to AcceptanceTests/torch/staging/test_ssd_mobilenetv2_quanteval.py diff --git a/AcceptanceTests/torch/test_ssd_res50_quanteval.py b/AcceptanceTests/torch/staging/test_ssd_res50_quanteval.py similarity index 97% rename from AcceptanceTests/torch/test_ssd_res50_quanteval.py rename to AcceptanceTests/torch/staging/test_ssd_res50_quanteval.py index e259fa1..ec740a4 100644 --- a/AcceptanceTests/torch/test_ssd_res50_quanteval.py +++ b/AcceptanceTests/torch/staging/test_ssd_res50_quanteval.py @@ -30,6 +30,7 @@ def test_quaneval_ssd_res50(model_config, tiny_mscoco_validation_path): model_config, "--dataset-path", tiny_mscoco_validation_path, + "--use-cuda" ] ) diff --git a/AcceptanceTests/torch/test_uniformer_classification_quanteval.py b/AcceptanceTests/torch/staging/test_uniformer_classification_quanteval.py similarity index 100% rename from AcceptanceTests/torch/test_uniformer_classification_quanteval.py rename to AcceptanceTests/torch/staging/test_uniformer_classification_quanteval.py diff --git a/AcceptanceTests/torch/test_deeplabv3_quanteval.py b/AcceptanceTests/torch/test_deeplabv3_quanteval.py index ad5db6b..92b13cd 100644 --- a/AcceptanceTests/torch/test_deeplabv3_quanteval.py +++ b/AcceptanceTests/torch/test_deeplabv3_quanteval.py @@ -9,9 +9,7 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= - """ acceptance test for deeplabv3 """ - import pytest from aimet_zoo_torch.deeplabv3.evaluators.deeplabv3_quanteval import main @@ -30,17 +28,18 @@ @pytest.mark.semantic_segmentation @pytest.mark.parametrize("model_config, expected_results", [('dlv3_w4a8', expected_results['dlv3_w4a8']), ('dlv3_w8a8', expected_results['dlv3_w8a8'])]) +@pytest.mark.skip(reason="Mini Dataset for deeplabv3 not set yet") def test_deeplabv3_quanteval( model_config, expected_results, PascalVOC_segmentation_test_data_path ): if PascalVOC_segmentation_test_data_path is None: - pytest.fail(f"dataset path is None!") + pytest.xfail(f"dataset path is None!") args = ['--model-config', model_config, - '--dataset-path', PascalVOC_segmentation_test_data_path.as_posix()] + '--dataset-path', PascalVOC_segmentation_test_data_path] mIoUs = main(args) - assert mIoUs['original_mIoU'] == expected_results['original_mIoU'] - assert mIoUs['quantized_mIoU'] == expected_results['quantized_mIoU'] + assert mIoUs['mIoU_orig_fp32'] == expected_results['original_mIoU'] + assert mIoUs['mIoU_optim_int8'] == expected_results['quantized_mIoU'] diff --git a/AcceptanceTests/torch/test_ffnet_quanteval.py b/AcceptanceTests/torch/test_ffnet_quanteval.py deleted file mode 100644 index 08b33a9..0000000 --- a/AcceptanceTests/torch/test_ffnet_quanteval.py +++ /dev/null @@ -1,46 +0,0 @@ -# /usr/bin/env python3 -# -*- mode: python -*- -# ============================================================================= -# @@-COPYRIGHT-START-@@ -# -# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. -# Changes from QuIC are licensed under the terms and conditions at -# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf -# -# @@-COPYRIGHT-END-@@ -# ============================================================================= - -""" acceptance test for ffnet semantic segmentation""" - -import pytest -import torch - -from aimet_zoo_torch.ffnet.evaluators import ( - ffnet_quanteval, -) - -@pytest.mark.sementic_segmentation -@pytest.mark.cuda -#pylint:disable = redefined-outer-name -@pytest.mark.parametrize( - "model_config",[ - "segmentation_ffnet40S_dBBB_mobile", - "segmentation_ffnet40S_dBBB_mobile", - "segmentation_ffnet78S_BCC_mobile_pre_down", - "segmentation_ffnet78S_BCC_mobile_pre_down", - "segmentation_ffnet122NS_CCC_mobile_pre_down" - ] - ) -def test_quaneval_ffnet(model_config, tiny_cityscapes_path): - """acceptance test of hrnet for semantic segmentation""" - torch.cuda.empty_cache() - if tiny_cityscapes_path is None: - pytest.fail('Dataset is not set') - ffnet_quanteval.main( - [ - "--model-config", - model_config, - "--dataset-path", - tiny_cityscapes_path, - ] - ) diff --git a/AcceptanceTests/torch/test_gpunet0_quanteval.py b/AcceptanceTests/torch/test_gpunet0_quanteval.py index 39f9ed3..2062a9e 100644 --- a/AcceptanceTests/torch/test_gpunet0_quanteval.py +++ b/AcceptanceTests/torch/test_gpunet0_quanteval.py @@ -20,10 +20,10 @@ @pytest.mark.image_classification # pylint:disable = redefined-outer-name @pytest.mark.parametrize("model_config", ["gpunet0_w8a8"]) -def test_quanteval_gpunet0_image_classification(model_config, tiny_imageNet_validation_path): +def test_quanteval_gpunet0_image_classification(model_config, tiny_imageNet_root_path): """gpunet0 image classification test""" - if tiny_imageNet_validation_path is None: + if tiny_imageNet_root_path is None: pytest.fail(f'Dataset path is not set') torch.cuda.empty_cache() @@ -32,6 +32,6 @@ def test_quanteval_gpunet0_image_classification(model_config, tiny_imageNet_vali "--model-config", model_config, "--dataset-path", - tiny_imageNet_validation_path, + tiny_imageNet_root_path, ] ) diff --git a/Jenkins/Dockerfile.tf-torch-cpu b/Jenkins/Dockerfile.tf-torch-cpu index 6451d48..06c87c9 100644 --- a/Jenkins/Dockerfile.tf-torch-cpu +++ b/Jenkins/Dockerfile.tf-torch-cpu @@ -24,7 +24,7 @@ RUN apt-get update -y > /dev/null && \ python3-pip && \ rm -rf /var/lib/apt/lists/* -# Upgrade Python3 pip and install some more packages -RUN python3 -m pip --no-cache-dir install --upgrade \ - pip \ - pylint==2.17.2 +# Upgrade Python3 pip and install some more packages +RUN python3 -m pip --no-cache-dir install --upgrade pip && \ + pip install pylint==2.17.2 && \ + pip install astroid==2.15.2 diff --git a/Jenkins/jenkins_threshold_configs.json b/Jenkins/jenkins_threshold_configs.json index 5a1125c..30cc24c 100644 --- a/Jenkins/jenkins_threshold_configs.json +++ b/Jenkins/jenkins_threshold_configs.json @@ -1,8 +1,8 @@ { "pylint_fail_thresholds" : { - "high_priority" : "10", - "normal_priority" : "10", - "low_priority" : "33" + "high_priority" : "0", + "normal_priority" : "0", + "low_priority" : "0" } } diff --git a/NOTICE.txt b/NOTICE.txt index 58508d8..bee6f8b 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -414,7 +414,7 @@ POSSIBILITY OF SUCH DAMAGE. Copyright (c) 2022 Qualcomm Technologies, Inc. This software may be subject to U.S. and international export, re-export, or -transfer (“export”) laws. Diversion contrary to U.S. and international law is +transfer (“export? laws. Diversion contrary to U.S. and international law is strictly prohibited. All rights reserved. @@ -620,3 +620,27 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. # ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ------------------------------------------------------------------------------ \ No newline at end of file diff --git a/README.md b/README.md index d777424..dd3d64f 100755 --- a/README.md +++ b/README.md @@ -619,6 +619,29 @@ An original FP32 source model is quantized either using post-training quantizati 24.78 / 0.628 25.41 / 0.666 (INT8W / INT16Act.) + + Semantic Segmentation + DeeplabV3plus_mbnv2 + GitHub Repo + Pretrained Model + See Example + 2.4 + (PascalVOC) mIOU + 72.28 + 71.71 + TBD + + + DeeplabV3plus_xception + GitHub Repo + Pretrained Model + See Example + 2.4 + (PascalVOC) mIOU + 87.71 + 87.21 + TBD + *[1]* Model usage documentation diff --git a/aimet_zoo_tensorflow/common/downloader.py b/aimet_zoo_tensorflow/common/downloader.py index 0edf829..7c09f70 100644 --- a/aimet_zoo_tensorflow/common/downloader.py +++ b/aimet_zoo_tensorflow/common/downloader.py @@ -3,11 +3,11 @@ # ============================================================================= # @@-COPYRIGHT-START-@@ # -# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Copyright (c) 2022 of Qualcomm Innovation Center, Inc. All rights reserved. # # @@-COPYRIGHT-END-@@ # ============================================================================= -""" Downloader and downloader progress bar class for downloading and loading weights and encodings""" +""" Downloader and downlowder progress bar class for downloading and loading weights and encodings""" import os import tarfile @@ -15,15 +15,18 @@ import shutil from shutil import copy2 from urllib.request import urlretrieve +import urllib.request import progressbar -import gdown # pylint: disable=import-error +import requests +import gdown# pylint: disable=import-error + class Downloader: """ Parent class for inheritance of utility methods used for downloading and loading weights and encodings """ - # pylint: disable=too-many-instance-attributes, too-many-arguments + # pylint: disable=too-many-instance-attributes # 16 is reasonable in this case. def __init__( self, @@ -37,7 +40,7 @@ def __init__( url_zipped_checkpoint: str = None, model_dir: str = "", model_config: str = "", - ): + ):# pylint: disable=too-many-arguments """ :param url_pre_opt_weights: url hosting pre optimization weights as a state dict :param url_post_opt_weights: url hosting post optimization weights as a state dict @@ -62,32 +65,32 @@ def __init__( else Path(model_dir + "/weights/") ) self.path_pre_opt_weights = ( - os.path.join(self._download_storage_path, "pre_opt_weights") + str(self._download_storage_path) + "/pre_opt_weights" if self.url_pre_opt_weights else None ) self.path_post_opt_weights = ( - os.path.join(self._download_storage_path, "post_opt_weights") + str(self._download_storage_path) + "/post_opt_weights" if self.url_post_opt_weights else None ) self.path_adaround_encodings = ( - os.path.join(self._download_storage_path, "adaround_encodings") + str(self._download_storage_path) + "/adaround_encodings" if self.url_adaround_encodings else None ) self.path_aimet_encodings = ( - os.path.join(self._download_storage_path, "aimet_encodings") + str(self._download_storage_path) + "/aimet_encodings" if self.url_aimet_encodings else None ) self.path_aimet_config = ( - os.path.join(self._download_storage_path, "aimet_config") + str(self._download_storage_path) + "/aimet_config" if self.url_aimet_config else None ) self.path_zipped_checkpoint = ( - os.path.join(self._download_storage_path, "zipped_checkpoint.zip") + str(self._download_storage_path) + "/zipped_checkpoint.zip" if self.url_zipped_checkpoint else None ) @@ -96,9 +99,14 @@ def __init__( if self.path_zipped_checkpoint else None ) + # GITHUB TOKEN for internal use cases + self.GITHUB_TOKEN = None + self.INTERNAL_REPO_URL = None - def _download_from_url(self, src: str, dst: str): + def _download_from_url(self, src: str, dst: str, show_progress=False): """Receives a source URL or path and a storage destination path, evaluates the source, fetches the file, and stores at the destination""" + # import pdb + # pdb.set_trace() if not os.path.exists(self._download_storage_path): os.makedirs(self._download_storage_path) if src is None: @@ -106,7 +114,13 @@ def _download_from_url(self, src: str, dst: str): if src.startswith("https://drive.google.com"): gdown.download(url=src, output=dst, quiet=True, verify=False) elif src.startswith("http"): - urlretrieve(src, dst) + if 'qualcomm' in src: + self._download_from_internal(src,dst) + else: + if show_progress: + urlretrieve(src, dst, DownloadProgressBar()) + else: + urlretrieve(src, dst) else: assert os.path.exists( src @@ -114,16 +128,75 @@ def _download_from_url(self, src: str, dst: str): copy2(src, dst) return None - def _download_pre_opt_weights(self): + def _convert_src_to_asset_url(self, src: str): + """convert src url to asset url + """ + # 0. get release_tag and file_name from url + release_tag, file_name = self._find_tag(src) + # 1. read all release in to all_releases + headers = { + 'Authorization': 'token ' + self.GITHUB_TOKEN , + 'Accept': 'application/json', + } + + resp = requests.get(self.INTERNAL_REPO_URL,headers = headers,timeout=(4, 30)) + + all_releases = resp.json() + # 2. check if release_tag in all_releases else report artifacts not uploade + content_with_tag_name = [s for s in all_releases if s['tag_name']== release_tag ] + if content_with_tag_name is None: + raise NameError('this release tag is not uploaded, check if release tag or if this release is uploaded yet') + # 3. check if file_name in all_releases['release_tag'], else report file not uploaded or file name is wrong + assets_with_tag_name = content_with_tag_name[0]['assets'] + asset_with_file_name = [s for s in assets_with_tag_name if s['name']== file_name ] + if asset_with_file_name is None: + raise NameError('this artifact is not uploaded or naming has mismatch with release') + # 4. return asset_url + return asset_with_file_name[0]['url'] + + def _find_tag(self, src: str): + """find out release tag and file name + /download/tensorflow2_resnet50/resnet50_w8a8.encodings + return should be + tensorflow2_resnet50, resnet50_w8a8.encodings + """ + url_breakdown = src.split('/') + return url_breakdown[-2], url_breakdown[-1] + + def _download_from_internal(self, src: str, dst: str): + """Use GITHUB_TOKEN evironment variable to download from internal github repo link + + """ + self.GITHUB_TOKEN= os.getenv("GITHUB_TOKEN") + self.INTERNAL_REPO_URL= os.getenv("INTERNAL_REPO_URL") + if self.GITHUB_TOKEN is None: + raise NameError("GITHUB_TOKEN not setup, not able to download from internal github url, exit program!") + if self.INTERNAL_REPO_URL is None: + raise NameError("variable INTERNAL_REPO_URL not setup, use export INTERNAL_REPO_URL= to setup before continuing") + asset_url = self._convert_src_to_asset_url(src) + headers = { + 'Authorization': 'token ' + self.GITHUB_TOKEN , + 'Accept': 'application/octet-stream', + } + resp = requests.get(asset_url,headers = headers, timeout=(4, 30) ) + with open(dst, 'wb') as file: + file.write(resp.content) + + + def _download_pre_opt_weights(self, show_progress=False): """downloads pre optimization weights""" self._download_from_url( - src=self.url_pre_opt_weights, dst=self.path_pre_opt_weights + src=self.url_pre_opt_weights, + dst=self.path_pre_opt_weights, + show_progress=show_progress, ) - def _download_post_opt_weights(self): + def _download_post_opt_weights(self, show_progress=False): """downloads post optimization weights""" self._download_from_url( - src=self.url_post_opt_weights, dst=self.path_post_opt_weights + src=self.url_post_opt_weights, + dst=self.path_post_opt_weights, + show_progress=show_progress, ) def _download_adaround_encodings(self): @@ -169,7 +242,7 @@ def _download_tar_decompress(self, tar_url): download_tar_name = ( str(self._download_storage_path) + "/downloaded_weights.tar.gz" ) - urlretrieve(tar_url, download_tar_name, DownloadProgressBar()) + urllib.request.urlretrieve(tar_url, download_tar_name, DownloadProgressBar()) with tarfile.open(download_tar_name) as pth_weights: pth_weights.extractall(self._download_storage_path) folder_name = pth_weights.getnames()[0] diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/Deeplabv3plus_TF2.md b/aimet_zoo_tensorflow/deeplabv3plus_tf2/Deeplabv3plus_TF2.md new file mode 100644 index 0000000..d2d9428 --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/Deeplabv3plus_TF2.md @@ -0,0 +1,74 @@ +# Tensorflow DeeplabV3plus for TensorFlow 2.4 + +## Setup AI Model Efficiency Toolkit (AIMET) +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.25/packaging/install.md) before proceeding further. This evaluation was run using [AIMET 1.25 for TensorFlow 2.4](https://github.com/quic/aimet/releases/tag/1.25.0) i.e. please set `release_tag="1.25"` and `AIMET_VARIANT="tf_gpu"` in the above instructions. + +## Additional Dependencies +pip install matplotlib==3.2.1 + +## Model checkpoint and dataset +The TF2 pretrained DeeplabV3plus can be downloaded at release page + +## Dataset +The Pascal Dataset can be downloaded from here: + - http://host.robots.ox.ac.uk/pascal/VOC/voc2012/ + +## Usage +- To run evaluation with QuantSim in AIMET, use the following: +```bash +python aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/deeplabv3plus_tf2_quanteval.py \ + --dataset-path \ + --batch-size \ + --model-config +``` +Available model configurations are: + +- deeplabv3plus_xception_w8a8 + +- deeplabv3plus_mbnv2_w8a8 + +- Example : python aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/deeplabv3plus_tf2_quanteval.py --dataset-path --batch-size 4 --model-config deeplabv3plus_xception_w8a8 + +## Quantization configuration +In the evaluation script included, we have used the default config file, which configures the quantizer ops with the following assumptions: +- Weight quantization: 8 bits, symmetric quantization +- Bias parameters are not quantized +- Activation quantization: 8 bits, asymmetric quantization +- Model inputs are quantized +- For MobileNetV2 backbone, Quantization Aware Traning has been performed on the optimized checkpoint + +## Results +Below are the *mIoU* results of the TensorFlow 2 DeeplabV3plus_xception model for the VOC2012 dataset: + + + + + + + + + + + + + + +
Model ConfigurationmIoU (%)
DeeplabV3plus_xception_FP3287.71
DeeplabV3plus_xception + simple PTQ(w8a8)87.21
+ +Below are the *mIoU* results of the TensorFlow 2 DeeplabV3plus_mbnv2 model for the VOC2012 dataset: + + + + + + + + + + + + + + +
Model ConfigurationmIoU (%)
DeeplabV3plus_mbnv2_FP3272.28
DeeplabV3plus_mbnv2 + QAT(w8a8)71.71
+ diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/dataloader/dataloader.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/dataloader/dataloader.py new file mode 100644 index 0000000..a53f6ca --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/dataloader/dataloader.py @@ -0,0 +1,228 @@ +# /usr/bin/env python3 +# -*- mode: python -*- + +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +import math +import os +from random import shuffle + +import cv2 +import numpy as np +from PIL import Image +from tensorflow import keras + +from aimet_zoo_tensorflow.deeplabv3plus_tf2.dataloader.utils import cvtColor, preprocess_input + + +class DeeplabDataset(keras.utils.Sequence): + def __init__(self, annotation_lines, input_shape, batch_size, num_classes, train, dataset_path): + self.annotation_lines = annotation_lines + self.length = len(self.annotation_lines) + self.input_shape = input_shape + self.batch_size = batch_size + self.num_classes = num_classes + self.train = train + self.dataset_path = dataset_path + + def __len__(self): + return math.ceil(len(self.annotation_lines) / float(self.batch_size)) + + def __getitem__(self, index): + images = [] + targets = [] + for i in range(index * self.batch_size, (index + 1) * self.batch_size): + i = i % self.length + name = self.annotation_lines[i].split()[0] + #-------------------------------# + # ļжȡͼ + #-------------------------------# + jpg = Image.open(os.path.join(os.path.join(self.dataset_path, "VOC2012/JPEGImages"), name + ".jpg")) + png = Image.open(os.path.join(os.path.join(self.dataset_path, "VOC2012/SegmentationClass"), name + ".png")) + #-------------------------------# + # ǿ + #-------------------------------# + jpg, png = self.get_random_data(jpg, png, self.input_shape, random = self.train) + jpg = preprocess_input(np.array(jpg, np.float32)) + png = np.array(png) + png[png >= self.num_classes] = self.num_classes + #-------------------------------------------------------# + # תone_hotʽ + # Ҫ+1ΪvocݼЩǩаױ߲ + # Ҫױֽ߲кԣ+1ĿǷԡ + #-------------------------------------------------------# + seg_labels = np.eye(self.num_classes + 1)[png.reshape([-1])] + seg_labels = seg_labels.reshape((int(self.input_shape[0]), int(self.input_shape[1]), self.num_classes + 1)) + + images.append(jpg) + targets.append(seg_labels) + + images = np.array(images) + targets = np.array(targets) + return images, targets + + def generate(self): + i = 0 + while True: + images = [] + targets = [] + for b in range(self.batch_size): + if i==0: + np.random.shuffle(self.annotation_lines) + name = self.annotation_lines[i].split()[0] + #-------------------------------# + # ļжȡͼ + #-------------------------------# + jpg = Image.open(os.path.join(os.path.join(self.dataset_path, "VOC2012/JPEGImages"), name + ".jpg")) + png = Image.open(os.path.join(os.path.join(self.dataset_path, "VOC2012/SegmentationClass"), name + ".png")) + #-------------------------------# + # ǿ + #-------------------------------# + jpg, png = self.get_random_data(jpg, png, self.input_shape, random = self.train) + jpg = preprocess_input(np.array(jpg, np.float32)) + png = np.array(png) + png[png >= self.num_classes] = self.num_classes + #-------------------------------------------------------# + # תone_hotʽ + # Ҫ+1ΪvocݼЩǩаױ߲ + # Ҫױֽ߲кԣ+1ĿǷԡ + #-------------------------------------------------------# + seg_labels = np.eye(self.num_classes + 1)[png.reshape([-1])] + seg_labels = seg_labels.reshape((int(self.input_shape[0]), int(self.input_shape[1]), self.num_classes + 1)) + + images.append(jpg) + targets.append(seg_labels) + i = (i + 1) % self.length + + images = np.array(images) + targets = np.array(targets) + yield images, targets + + def on_epoch_end(self): + shuffle(self.annotation_lines) + + def rand(self, a=0, b=1): + return np.random.rand() * (b - a) + a + + def get_random_data(self, image, label, input_shape, jitter=.3, hue=.1, sat=1.5, val=1.5, random=True): + image = cvtColor(image) + label = Image.fromarray(np.array(label)) + h, w = input_shape + + if not random: + iw, ih = image.size + scale = min(w/iw, h/ih) + nw = int(iw*scale) + nh = int(ih*scale) + + image = image.resize((nw,nh), Image.BICUBIC) + new_image = Image.new('RGB', [w, h], (128,128,128)) + new_image.paste(image, ((w-nw)//2, (h-nh)//2)) + + label = label.resize((nw,nh), Image.NEAREST) + new_label = Image.new('L', [w, h], (0)) + new_label.paste(label, ((w-nw)//2, (h-nh)//2)) + return new_image, new_label + + # resize image + rand_jit1 = self.rand(1-jitter,1+jitter) + rand_jit2 = self.rand(1-jitter,1+jitter) + new_ar = w/h * rand_jit1/rand_jit2 + + scale = self.rand(0.25, 2) + if new_ar < 1: + nh = int(scale*h) + nw = int(nh*new_ar) + else: + nw = int(scale*w) + nh = int(nw/new_ar) + + image = image.resize((nw,nh), Image.BICUBIC) + label = label.resize((nw,nh), Image.NEAREST) + + flip = self.rand()<.5 + if flip: + image = image.transpose(Image.FLIP_LEFT_RIGHT) + label = label.transpose(Image.FLIP_LEFT_RIGHT) + + # place image + dx = int(self.rand(0, w-nw)) + dy = int(self.rand(0, h-nh)) + new_image = Image.new('RGB', (w,h), (128,128,128)) + new_label = Image.new('L', (w,h), (0)) + new_image.paste(image, (dx, dy)) + new_label.paste(label, (dx, dy)) + image = new_image + label = new_label + + image_data = np.array(image, np.uint8) + + #------------------------------------------# + # ˹ģ + #------------------------------------------# + blur = self.rand() < 0.25 + if blur: + image_data = cv2.GaussianBlur(image_data, (5, 5), 0) + + #------------------------------------------# + # ת + #------------------------------------------# + rotate = self.rand() < 0.25 + if rotate: + center = (w // 2, h // 2) + rotation = np.random.randint(-10, 11) + M = cv2.getRotationMatrix2D(center, -rotation, scale=1) + image_data = cv2.warpAffine(image_data, M, (w, h), flags=cv2.INTER_CUBIC, borderValue=(128,128,128)) + label = cv2.warpAffine(np.array(label, np.uint8), M, (w, h), flags=cv2.INTER_NEAREST, borderValue=(0)) + + #---------------------------------# + # ͼɫ任 + # ɫ任IJ + #---------------------------------# + r = np.random.uniform(-1, 1, 3) * [hue, sat, val] + 1 + #---------------------------------# + # ͼתHSV + #---------------------------------# + hue, sat, val = cv2.split(cv2.cvtColor(image_data, cv2.COLOR_RGB2HSV)) + dtype = image_data.dtype + #---------------------------------# + # Ӧñ任 + #---------------------------------# + x = np.arange(0, 256, dtype=r.dtype) + lut_hue = ((x * r[0]) % 180).astype(dtype) + lut_sat = np.clip(x * r[1], 0, 255).astype(dtype) + lut_val = np.clip(x * r[2], 0, 255).astype(dtype) + + image_data = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))) + image_data = cv2.cvtColor(image_data, cv2.COLOR_HSV2RGB) + + return image_data, label \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/dataloader/utils.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/dataloader/utils.py new file mode 100644 index 0000000..65adf19 --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/dataloader/utils.py @@ -0,0 +1,282 @@ +# /usr/bin/env python3 +# -*- mode: python -*- + +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +import numpy as np +from PIL import Image + +#---------------------------------------------------------# +# ͼתRGBͼ񣬷ֹҶͼԤʱ +# ֧RGBͼԤ⣬͵ͼ񶼻תRGB +#---------------------------------------------------------# +def cvtColor(image): + if len(np.shape(image)) == 3 and np.shape(image)[2] == 3: + return image + else: + image = image.convert('RGB') + return image + +#---------------------------------------------------# +# ͼresize +#---------------------------------------------------# +def resize_image(image, size): + iw, ih = image.size + w, h = size + + scale = min(w/iw, h/ih) + nw = int(iw*scale) + nh = int(ih*scale) + + image = image.resize((nw,nh), Image.BICUBIC) + new_image = Image.new('RGB', size, (128,128,128)) + new_image.paste(image, ((w-nw)//2, (h-nh)//2)) + + return new_image, nw, nh + +def preprocess_input(image): + image = image / 127.5 - 1 + return image + +def show_config(**kwargs): + print('Configurations:') + print('-' * 70) + print('|%25s | %40s|' % ('keys', 'values')) + print('-' * 70) + for key, value in kwargs.items(): + print('|%25s | %40s|' % (str(key), str(value))) + print('-' * 70) + +#-------------------------------------------------------------------------------------------------------------------------------# +# From https://github.com/ckyrkou/Keras_FLOP_Estimator +# Fix lots of bugs +#-------------------------------------------------------------------------------------------------------------------------------# +def net_flops(model, table=False, print_result=True): + if (table == True): + print("\n") + print('%25s | %16s | %16s | %16s | %16s | %6s | %6s' % ( + 'Layer Name', 'Input Shape', 'Output Shape', 'Kernel Size', 'Filters', 'Strides', 'FLOPS')) + print('=' * 120) + + #---------------------------------------------------# + # ܵFLOPs + #---------------------------------------------------# + t_flops = 0 + factor = 1e9 + + for l in model.layers: + try: + #--------------------------------------# + # ijʼ + #--------------------------------------# + o_shape, i_shape, strides, ks, filters = ('', '', ''), ('', '', ''), (1, 1), (0, 0), 0 + flops = 0 + #--------------------------------------# + # ò + #--------------------------------------# + name = l.name + + if ('InputLayer' in str(l)): + i_shape = l.get_input_shape_at(0)[1:4] + o_shape = l.get_output_shape_at(0)[1:4] + + #--------------------------------------# + # Reshape + #--------------------------------------# + elif ('Reshape' in str(l)): + i_shape = l.get_input_shape_at(0)[1:4] + o_shape = l.get_output_shape_at(0)[1:4] + + #--------------------------------------# + # + #--------------------------------------# + elif ('Padding' in str(l)): + i_shape = l.get_input_shape_at(0)[1:4] + o_shape = l.get_output_shape_at(0)[1:4] + + #--------------------------------------# + # ƽ̲ + #--------------------------------------# + elif ('Flatten' in str(l)): + i_shape = l.get_input_shape_at(0)[1:4] + o_shape = l.get_output_shape_at(0)[1:4] + + #--------------------------------------# + #  + #--------------------------------------# + elif 'Activation' in str(l): + i_shape = l.get_input_shape_at(0)[1:4] + o_shape = l.get_output_shape_at(0)[1:4] + + #--------------------------------------# + # LeakyReLU + #--------------------------------------# + elif 'LeakyReLU' in str(l): + for i in range(len(l._inbound_nodes)): + i_shape = l.get_input_shape_at(i)[1:4] + o_shape = l.get_output_shape_at(i)[1:4] + + flops += i_shape[0] * i_shape[1] * i_shape[2] + + #--------------------------------------# + # ػ + #--------------------------------------# + elif 'MaxPooling' in str(l): + i_shape = l.get_input_shape_at(0)[1:4] + o_shape = l.get_output_shape_at(0)[1:4] + + #--------------------------------------# + # ػ + #--------------------------------------# + elif ('AveragePooling' in str(l) and 'Global' not in str(l)): + strides = l.strides + ks = l.pool_size + + for i in range(len(l._inbound_nodes)): + i_shape = l.get_input_shape_at(i)[1:4] + o_shape = l.get_output_shape_at(i)[1:4] + + flops += o_shape[0] * o_shape[1] * o_shape[2] + + #--------------------------------------# + # ȫֳػ + #--------------------------------------# + elif ('AveragePooling' in str(l) and 'Global' in str(l)): + for i in range(len(l._inbound_nodes)): + i_shape = l.get_input_shape_at(i)[1:4] + o_shape = l.get_output_shape_at(i)[1:4] + + flops += (i_shape[0] * i_shape[1] + 1) * i_shape[2] + + #--------------------------------------# + # ׼ + #--------------------------------------# + elif ('BatchNormalization' in str(l)): + for i in range(len(l._inbound_nodes)): + i_shape = l.get_input_shape_at(i)[1:4] + o_shape = l.get_output_shape_at(i)[1:4] + + temp_flops = 1 + for i in range(len(i_shape)): + temp_flops *= i_shape[i] + temp_flops *= 2 + + flops += temp_flops + + #--------------------------------------# + # ȫӲ + #--------------------------------------# + elif ('Dense' in str(l)): + for i in range(len(l._inbound_nodes)): + i_shape = l.get_input_shape_at(i)[1:4] + o_shape = l.get_output_shape_at(i)[1:4] + + temp_flops = 1 + for i in range(len(o_shape)): + temp_flops *= o_shape[i] + + if (i_shape[-1] == None): + temp_flops = temp_flops * o_shape[-1] + else: + temp_flops = temp_flops * i_shape[-1] + flops += temp_flops + + #--------------------------------------# + # ͨ + #--------------------------------------# + elif ('Conv2D' in str(l) and 'DepthwiseConv2D' not in str(l) and 'SeparableConv2D' not in str(l)): + strides = l.strides + ks = l.kernel_size + filters = l.filters + bias = 1 if l.use_bias else 0 + + for i in range(len(l._inbound_nodes)): + i_shape = l.get_input_shape_at(i)[1:4] + o_shape = l.get_output_shape_at(i)[1:4] + + if (filters == None): + filters = i_shape[2] + flops += filters * o_shape[0] * o_shape[1] * (ks[0] * ks[1] * i_shape[2] + bias) + + #--------------------------------------# + # + #--------------------------------------# + elif ('Conv2D' in str(l) and 'DepthwiseConv2D' in str(l) and 'SeparableConv2D' not in str(l)): + strides = l.strides + ks = l.kernel_size + filters = l.filters + bias = 1 if l.use_bias else 0 + + for i in range(len(l._inbound_nodes)): + i_shape = l.get_input_shape_at(i)[1:4] + o_shape = l.get_output_shape_at(i)[1:4] + + if (filters == None): + filters = i_shape[2] + flops += filters * o_shape[0] * o_shape[1] * (ks[0] * ks[1] + bias) + + #--------------------------------------# + # ȿɷ + #--------------------------------------# + elif ('Conv2D' in str(l) and 'DepthwiseConv2D' not in str(l) and 'SeparableConv2D' in str(l)): + strides = l.strides + ks = l.kernel_size + filters = l.filters + + for i in range(len(l._inbound_nodes)): + i_shape = l.get_input_shape_at(i)[1:4] + o_shape = l.get_output_shape_at(i)[1:4] + + if (filters == None): + filters = i_shape[2] + flops += i_shape[2] * o_shape[0] * o_shape[1] * (ks[0] * ks[1] + bias) + \ + filters * o_shape[0] * o_shape[1] * (1 * 1 * i_shape[2] + bias) + #--------------------------------------# + # ģģʱ + #--------------------------------------# + elif 'Model' in str(l): + flops = net_flops(l, print_result=False) + + t_flops += flops + + if (table == True): + print('%25s | %16s | %16s | %16s | %16s | %6s | %5.4f' % ( + name[:25], str(i_shape), str(o_shape), str(ks), str(filters), str(strides), flops)) + + except: + pass + + t_flops = t_flops * 2 + if print_result: + show_flops = t_flops / factor + print('Total GFLOPs: %.3fG' % (show_flops)) + return t_flops \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/deeplabv3plus_tf2_quanteval.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/deeplabv3plus_tf2_quanteval.py new file mode 100644 index 0000000..d7623b7 --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/deeplabv3plus_tf2_quanteval.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2022 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +''' do TF2 deeplabv3plus quantization and evaluation''' +import argparse +import tensorflow as tf +from aimet_zoo_tensorflow.deeplabv3plus_tf2.model.model_definition import Deeplabv3Plus +from aimet_zoo_tensorflow.deeplabv3plus_tf2.evaluators.eval_func import get_eval_func + +def arguments(raw_args): + ''' + parses command line arguments + ''' + parser = argparse.ArgumentParser(description='Arguments for evaluating model') + parser.add_argument('--dataset-path', help='path to image evaluation dataset', type=str) + parser.add_argument('--model-config', help='model configuration to be tested', type=str) + parser.add_argument('--default-output-bw', help='Default output bitwidth for quantization.', type=int, default=8) + parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) + parser.add_argument('--batch-size', help='batch_size for loading data', type=int, default=4) + parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) + args = parser.parse_args(raw_args) + return args + +def main(raw_args=None): + """ Run evaluation """ + args = arguments(raw_args) + + gpu_devices = tf.config.experimental.list_physical_devices("GPU") + for device in gpu_devices: + tf.config.experimental.set_memory_growth(device, True) + + + # Evaluation function + eval_func = get_eval_func(dataset_dir=args.dataset_path, + batch_size=args.batch_size, + num_iterations=5000) + + # Models + model = Deeplabv3Plus(model_config = args.model_config) + fp32_acc = 0 + quant_acc = 0 + + # For xception backbone + if 'xception' in args.model_config: + + model.from_pretrained(quantized=True) + sim = model.get_quantsim(quantized=True) + + # Evaluate original + print("start evaluating FP32 accuracy") + fp32_acc = eval_func(model.model) + print(f'FP32 top1 accuracy: {fp32_acc:0.3f}') + + # Evaluate optimized + print("start evaluating quantized accuracy") + quant_acc = eval_func(sim.model) + print(f'Quantized top1 accuracy: {quant_acc:0.3f}') + + # For mobilenetv2 backbone + elif 'mbnv2' in args.model_config: + + model.from_pretrained(quantized=False) + # Evaluate original + print("start evaluating FP32 accuracy") + fp32_acc = eval_func(model.model) + print(f'FP32 top1 accuracy: {fp32_acc:0.3f}') + + model.from_pretrained(quantized=True) + sim = model.get_quantsim(quantized=True) + + # Evaluate optimized + print("start evaluating quantized accuracy") + quant_acc = eval_func(sim.model) + print(f'Quantized top1 accuracy: {quant_acc:0.3f}') + + else: + print("please check the model config filename") + + return {'fp32_acc':fp32_acc, 'quant_acc':quant_acc} + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/eval_func.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/eval_func.py new file mode 100644 index 0000000..1a0ad93 --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/eval_func.py @@ -0,0 +1,79 @@ +# /usr/bin/env python3 +# -*- mode: python -*- + +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +'''get evaluation function''' + +import os +import numpy as np +from tqdm import tqdm +import tensorflow as tf +from aimet_zoo_tensorflow.deeplabv3plus_tf2.dataloader.dataloader import DeeplabDataset +from aimet_zoo_tensorflow.deeplabv3plus_tf2.evaluators.utils_metrics import fast_hist, per_class_iu + +def get_eval_func(dataset_dir, batch_size, num_iterations=5000): + ''' + :param dataset_dir: data path + :param batch_size: batch size in evaluation and calibration + :param num_iterations: number of images used + :return evaluation function + ''' + def func_wrapper(model, iterations=num_iterations): + ''' + :param model: FP32 model or sim.model + :param iterations: number of images used + ''' + num_classes = 21 + # get validation dataset + with open(os.path.join(dataset_dir, "VOC2012/ImageSets/Segmentation/val.txt"),"r") as f: + val_lines = f.readlines() + val_dataloader = DeeplabDataset(val_lines, [512, 512], batch_size, num_classes, False, dataset_dir) + + # compute accuracy + hist = 0 + total = 0 + for index in tqdm(range(len(val_dataloader))): + images, labels = val_dataloader[index] + prediction = model(images, training=False) + labels = np.array(labels) + prediction = np.array(prediction) + labels = labels.argmax(-1) + prediction = prediction.argmax(-1) + hist += fast_hist(labels.flatten(), prediction.flatten(), num_classes) + total += batch_size + if total >= iterations: + break + IoUs = per_class_iu(hist) + return np.nanmean(IoUs) * 100 + + return func_wrapper \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/utils_metrics.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/utils_metrics.py new file mode 100644 index 0000000..4f8a4de --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/evaluators/utils_metrics.py @@ -0,0 +1,218 @@ +# /usr/bin/env python3 +# -*- mode: python -*- + +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +import csv +import os +from os.path import join + +import matplotlib.pyplot as plt +import numpy as np +from tensorflow.keras import backend +from PIL import Image + +def Iou_score(smooth = 1e-5, threhold = 0.5): + def _Iou_score(y_true, y_pred): + # score calculation + y_pred = backend.greater(y_pred, threhold) + y_pred = backend.cast(y_pred, backend.floatx()) + intersection = backend.sum(y_true[...,:-1] * y_pred, axis=[0,1,2]) + union = backend.sum(y_true[...,:-1] + y_pred, axis=[0,1,2]) - intersection + + score = (intersection + smooth) / (union + smooth) + return score + return _Iou_score + +def f_score(beta=1, smooth = 1e-5, threhold = 0.5): + def _f_score(y_true, y_pred): + y_pred = backend.greater(y_pred, threhold) + y_pred = backend.cast(y_pred, backend.floatx()) + + tp = backend.sum(y_true[...,:-1] * y_pred, axis=[0,1,2]) + fp = backend.sum(y_pred , axis=[0,1,2]) - tp + fn = backend.sum(y_true[...,:-1], axis=[0,1,2]) - tp + + score = ((1 + beta ** 2) * tp + smooth) \ + / ((1 + beta ** 2) * tp + beta ** 2 * fn + fp + smooth) + return score + return _f_score + +# ǩWH +def fast_hist(a, b, n): + #--------------------------------------------------------------------------------# + # aתһάıǩ״(HW,)bתһάԤ״(HW,) + #--------------------------------------------------------------------------------# + k = (a >= 0) & (a < n) + #--------------------------------------------------------------------------------# + # np.bincount˴0n**2-1n**2ÿֵĴֵ״(n, n) + # УдԽϵΪȷص + #--------------------------------------------------------------------------------# + return np.bincount(n * a[k].astype(int) + b[k], minlength=n ** 2).reshape(n, n) + +def per_class_iu(hist): + return np.diag(hist) / np.maximum((hist.sum(1) + hist.sum(0) - np.diag(hist)), 1) + +def per_class_PA_Recall(hist): + return np.diag(hist) / np.maximum(hist.sum(1), 1) + +def per_class_Precision(hist): + return np.diag(hist) / np.maximum(hist.sum(0), 1) + +def per_Accuracy(hist): + return np.sum(np.diag(hist)) / np.maximum(np.sum(hist), 1) + +def compute_mIoU(gt_dir, pred_dir, png_name_list, num_classes, name_classes=None): + print('Num classes', num_classes) + #-----------------------------------------# + # һȫ0ľһ + #-----------------------------------------# + hist = np.zeros((num_classes, num_classes)) + + #------------------------------------------------# + # ֤ǩ·бֱӶȡ + # ֤ͼָ·бֱӶȡ + #------------------------------------------------# + gt_imgs = [join(gt_dir, x + ".png") for x in png_name_list] + pred_imgs = [join(pred_dir, x + ".png") for x in png_name_list] + + #------------------------------------------------# + # ȡÿһͼƬ-ǩ + #------------------------------------------------# + for ind in range(len(gt_imgs)): + #------------------------------------------------# + # ȡһͼָתnumpy + #------------------------------------------------# + pred = np.array(Image.open(pred_imgs[ind])) + #------------------------------------------------# + # ȡһŶӦıǩתnumpy + #------------------------------------------------# + label = np.array(Image.open(gt_imgs[ind])) + + # ͼָǩĴСһͼƬͲ + if len(label.flatten()) != len(pred.flatten()): + print( + 'Skipping: len(gt) = {:d}, len(pred) = {:d}, {:s}, {:s}'.format( + len(label.flatten()), len(pred.flatten()), gt_imgs[ind], + pred_imgs[ind])) + continue + + #------------------------------------------------# + # һͼƬ2121hist󣬲ۼ + #------------------------------------------------# + hist += fast_hist(label.flatten(), pred.flatten(), num_classes) + # ÿ10žһĿǰѼͼƬƽmIoUֵ + if name_classes is not None and ind > 0 and ind % 10 == 0: + print('{:d} / {:d}: mIou-{:0.2f}%; mPA-{:0.2f}%; Accuracy-{:0.2f}%'.format( + ind, + len(gt_imgs), + 100 * np.nanmean(per_class_iu(hist)), + 100 * np.nanmean(per_class_PA_Recall(hist)), + 100 * per_Accuracy(hist) + ) + ) + #------------------------------------------------# + # ֤ͼƬmIoUֵ + #------------------------------------------------# + IoUs = per_class_iu(hist) + PA_Recall = per_class_PA_Recall(hist) + Precision = per_class_Precision(hist) + #------------------------------------------------# + # һmIoUֵ + #------------------------------------------------# + if name_classes is not None: + for ind_class in range(num_classes): + print('===>' + name_classes[ind_class] + ':\tIou-' + str(round(IoUs[ind_class] * 100, 2)) \ + + '; Recall (equal to the PA)-' + str(round(PA_Recall[ind_class] * 100, 2))+ '; Precision-' + str(round(Precision[ind_class] * 100, 2))) + + #-----------------------------------------------------------------# + # ֤ͼƽmIoUֵʱNaNֵ + #-----------------------------------------------------------------# + print('===> mIoU: ' + str(round(np.nanmean(IoUs) * 100, 2)) + '; mPA: ' + str(round(np.nanmean(PA_Recall) * 100, 2)) + '; Accuracy: ' + str(round(per_Accuracy(hist) * 100, 2))) + return np.array(hist, np.int), IoUs, PA_Recall, Precision + +def adjust_axes(r, t, fig, axes): + bb = t.get_window_extent(renderer=r) + text_width_inches = bb.width / fig.dpi + current_fig_width = fig.get_figwidth() + new_fig_width = current_fig_width + text_width_inches + propotion = new_fig_width / current_fig_width + x_lim = axes.get_xlim() + axes.set_xlim([x_lim[0], x_lim[1] * propotion]) + +def draw_plot_func(values, name_classes, plot_title, x_label, output_path, tick_font_size = 12, plt_show = True): + fig = plt.gcf() + axes = plt.gca() + plt.barh(range(len(values)), values, color='royalblue') + plt.title(plot_title, fontsize=tick_font_size + 2) + plt.xlabel(x_label, fontsize=tick_font_size) + plt.yticks(range(len(values)), name_classes, fontsize=tick_font_size) + r = fig.canvas.get_renderer() + for i, val in enumerate(values): + str_val = " " + str(val) + if val < 1.0: + str_val = " {0:.2f}".format(val) + t = plt.text(val, i, str_val, color='royalblue', va='center', fontweight='bold') + if i == (len(values)-1): + adjust_axes(r, t, fig, axes) + + fig.tight_layout() + fig.savefig(output_path) + if plt_show: + plt.show() + plt.close() + +def show_results(miou_out_path, hist, IoUs, PA_Recall, Precision, name_classes, tick_font_size = 12): + draw_plot_func(IoUs, name_classes, "mIoU = {0:.2f}%".format(np.nanmean(IoUs)*100), "Intersection over Union", \ + os.path.join(miou_out_path, "mIoU.png"), tick_font_size = tick_font_size, plt_show = True) + print("Save mIoU out to " + os.path.join(miou_out_path, "mIoU.png")) + + draw_plot_func(PA_Recall, name_classes, "mPA = {0:.2f}%".format(np.nanmean(PA_Recall)*100), "Pixel Accuracy", \ + os.path.join(miou_out_path, "mPA.png"), tick_font_size = tick_font_size, plt_show = False) + print("Save mPA out to " + os.path.join(miou_out_path, "mPA.png")) + + draw_plot_func(PA_Recall, name_classes, "mRecall = {0:.2f}%".format(np.nanmean(PA_Recall)*100), "Recall", \ + os.path.join(miou_out_path, "Recall.png"), tick_font_size = tick_font_size, plt_show = False) + print("Save Recall out to " + os.path.join(miou_out_path, "Recall.png")) + + draw_plot_func(Precision, name_classes, "mPrecision = {0:.2f}%".format(np.nanmean(Precision)*100), "Precision", \ + os.path.join(miou_out_path, "Precision.png"), tick_font_size = tick_font_size, plt_show = False) + print("Save Precision out to " + os.path.join(miou_out_path, "Precision.png")) + + with open(os.path.join(miou_out_path, "confusion_matrix.csv"), 'w', newline='') as f: + writer = csv.writer(f) + writer_list = [] + writer_list.append([' '] + [str(c) for c in name_classes]) + for i in range(len(hist)): + writer_list.append([name_classes[i]] + [str(x) for x in hist[i]]) + writer.writerows(writer_list) + print("Save confusion_matrix out to " + os.path.join(miou_out_path, "confusion_matrix.csv")) \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_cards/deeplabv3plus_mbnv2_w8a8.json b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_cards/deeplabv3plus_mbnv2_w8a8.json new file mode 100644 index 0000000..3aef874 --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_cards/deeplabv3plus_mbnv2_w8a8.json @@ -0,0 +1,24 @@ +{ + "name": "deeplabv3plus_mbnv2_tf2", + "framework": "tensorflow2.x", + "task": "semantic segmentation", + "input_shape": [512, 512, 3], + "dataset": "PascalVOC2012", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "techniques": ["bn_fold", "qat"] + } + }, + "artifacts": { + "url_pre_opt_weights": "https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/download/tensorflow2-deeplabv3plus_mbnv2/deeplabv3_mobilenetv2.h5", + "url_post_opt_weights": "https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/download/tensorflow2-deeplabv3plus_mbnv2/deeplabv3_mbnv2_w8a8.h5", + "url_adaround_encodings": null, + "url_aimet_encodings": "https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/download/tensorflow2-deeplabv3plus_mbnv2/deeplabv3_mbnv2_w8a8.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.25/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_cards/deeplabv3plus_xception_w8a8.json b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_cards/deeplabv3plus_xception_w8a8.json new file mode 100644 index 0000000..c6b90aa --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_cards/deeplabv3plus_xception_w8a8.json @@ -0,0 +1,24 @@ +{ + "name": "deeplabv3plus_xception_tf2", + "framework": "tensorflow2.x", + "task": "semantic segmentation", + "input_shape": [512, 512, 3], + "dataset": "PascalVOC2012", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "percentile", + "techniques": ["bn_fold"] + } + }, + "artifacts": { + "url_pre_opt_weights": "https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/download/tensorflow2-deeplabv3plus_xception/deeplabv3_xception.h5", + "url_post_opt_weights": null, + "url_adaround_encodings": null, + "url_aimet_encodings": "https://github.qualcomm.com/qualcomm-ai/aimet-model-zoo/releases/download/tensorflow2-deeplabv3plus_xception/deeplabv3_xception_w8a8.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.25/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_definition.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_definition.py new file mode 100644 index 0000000..cdc7210 --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/model_definition.py @@ -0,0 +1,140 @@ +# /usr/bin/env python3 +# -*- mode: python -*- + +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +''' Define Deeplabv3plus_xception model and do Quantsim''' +import os +import json +from aimet_tensorflow.keras.quantsim import QuantizationSimModel # pylint:disable = import-error +from aimet_tensorflow.keras.model_preparer import prepare_model # pylint:disable = import-error +from aimet_tensorflow.keras.batch_norm_fold import fold_all_batch_norms # pylint:disable = import-error +from aimet_zoo_tensorflow.deeplabv3plus_tf2.model.nets.deeplab import Deeplabv3 +from aimet_zoo_tensorflow.common.downloader import Downloader + +class Deeplabv3Plus(Downloader): + """Deeplabv3Plus_xception parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + # pylint: disable=unused-argument + def __init__(self, model_config = None, **kwargs): + """ + :param model_config: named model config from which to obtain model artifacts and arguments. + If provided, overwrites the other arguments passed to this object + """ + parent_dir = '/'.join(os.path.realpath(__file__).split('/')[:-1]) + self.cfg = False + if model_config: + config_filepath = parent_dir + '/model_cards/' + model_config + '.json' + if os.path.exists(config_filepath): + with open(config_filepath, encoding='UTF-8') as f_in: + self.cfg = json.load(f_in) + if self.cfg: + Downloader.__init__(self, + url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], + url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], + url_adaround_encodings = self.cfg['artifacts']['url_adaround_encodings'], + url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], + url_aimet_config = self.cfg['artifacts']['url_aimet_config'], + model_dir = parent_dir, + model_config = model_config) + self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) + + if 'xception' in model_config: + self.model = Deeplabv3(input_shape = [self.input_shape[0], self.input_shape[1], 3], + num_classes = 21, + backbone = 'xception') + elif 'mbnv2' in model_config: + self.model = Deeplabv3(input_shape = [self.input_shape[0], self.input_shape[1], 3], + num_classes = 21, + backbone = 'mobilenet') + else: + self.model = None + print("please check the model config filename") + + def from_pretrained(self, quantized=False): + """download config file and encodings from model cards""" + if not self.cfg: + raise NotImplementedError('There are no pretrained weights available for the model_config passed') + self._download_pre_opt_weights() + self._download_post_opt_weights() + self._download_aimet_config() + self._download_aimet_encodings() + self._download_adaround_encodings() + # load model weights + if self.path_pre_opt_weights: + self.model.load_weights(self.path_pre_opt_weights, by_name=True, skip_mismatch=True) + if quantized: + if self.path_post_opt_weights: + self.model.load_weights(self.path_post_opt_weights, by_name=True, skip_mismatch=True) + # model prepare + self.model = prepare_model(self.model) + # bn folding to weights + _, self.model = fold_all_batch_norms(self.model) + + + def get_quantsim(self, quantized=False): + """get quantsim object with pre-loaded encodings""" + if not self.cfg: + raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + if quantized: + self.from_pretrained(quantized=True) + else: + self.from_pretrained(quantized=False) + + kwargs = { + 'quant_scheme': self.cfg['optimization_config']['quantization_configuration']['quant_scheme'], + 'default_param_bw': self.cfg['optimization_config']['quantization_configuration']['param_bw'], + 'default_output_bw': self.cfg['optimization_config']['quantization_configuration']['output_bw'], + 'config_file': self.path_aimet_config, + } + + sim = QuantizationSimModel(self.model, **kwargs) + + if self.cfg['optimization_config']['quantization_configuration']['quant_scheme'] == "percentile": + sim.set_percentile_value(99.99) + + # load encoding file back to sim + if self.path_aimet_encodings and quantized: + sim.load_encodings_to_sim(self.path_aimet_encodings) + # This is the temporary solution for slow speed issue after loading_encoding_to_sim + # it will be removed once official solution available + # pylint: disable=protected-access + op_mode = sim._param_op_mode_after_analysis(sim.quant_scheme) + # pylint: disable=protected-access + sim._set_op_mode_parameters(op_mode) + print('load_encodings_to_sim finished!') + + # load adaround encoding file back to sim + if self.path_adaround_encodings and quantized: + sim.set_and_freeze_param_encodings(self.path_adaround_encodings) + print('set_and_freeze_param_encodings finished!') + + return sim \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/Xception.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/Xception.py new file mode 100644 index 0000000..73faa0b --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/Xception.py @@ -0,0 +1,173 @@ +# /usr/bin/env python3 +# -*- mode: python -*- + +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +from tensorflow.keras import layers +from tensorflow.keras.layers import (Activation, BatchNormalization, Conv2D, + DepthwiseConv2D, ZeroPadding2D) + + +def _conv2d_same(x, filters, prefix, stride=1, kernel_size=3, rate=1): + # paddinghwǷҪ + if stride == 1: + return Conv2D(filters, + (kernel_size, kernel_size), + strides=(stride, stride), + padding='same', use_bias=False, + dilation_rate=(rate, rate), + name=prefix)(x) + else: + kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) + pad_total = kernel_size_effective - 1 + pad_beg = pad_total // 2 + pad_end = pad_total - pad_beg + x = ZeroPadding2D((pad_beg, pad_end))(x) + return Conv2D(filters, + (kernel_size, kernel_size), + strides=(stride, stride), + padding='valid', use_bias=False, + dilation_rate=(rate, rate), + name=prefix)(x) + +def SepConv_BN(x, filters, prefix, stride=1, kernel_size=3, rate=1, depth_activation=False, epsilon=1e-3): + # paddinghwǷҪ + if stride == 1: + depth_padding = 'same' + else: + kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) + pad_total = kernel_size_effective - 1 + pad_beg = pad_total // 2 + pad_end = pad_total - pad_beg + x = ZeroPadding2D((pad_beg, pad_end))(x) + depth_padding = 'valid' + + # Ҫ + if not depth_activation: + x = Activation('relu')(x) + + # 3x31x1 + # 3x3; + x = DepthwiseConv2D((kernel_size, kernel_size), strides=(stride, stride), dilation_rate=(rate, rate), + padding=depth_padding, use_bias=False, name=prefix + '_depthwise')(x) + x = BatchNormalization(name=prefix + '_depthwise_BN', epsilon=epsilon)(x) + if depth_activation: + x = Activation('relu')(x) + + # 1x1ѹ + x = Conv2D(filters, (1, 1), padding='same', + use_bias=False, name=prefix + '_pointwise')(x) + x = BatchNormalization(name=prefix + '_pointwise_BN', epsilon=epsilon)(x) + if depth_activation: + x = Activation('relu')(x) + + return x + +def _xception_block(inputs, depth_list, prefix, skip_connection_type, stride, + rate=1, depth_activation=False, return_skip=False): + + residual = inputs + for i in range(3): + residual = SepConv_BN(residual, + depth_list[i], + prefix + '_separable_conv{}'.format(i + 1), + stride=stride if i == 2 else 1, + rate=rate, + depth_activation=depth_activation) + if i == 1: + skip = residual + if skip_connection_type == 'conv': + shortcut = _conv2d_same(inputs, depth_list[-1], prefix + '_shortcut', + kernel_size=1, + stride=stride) + shortcut = BatchNormalization(name=prefix + '_shortcut_BN')(shortcut) + outputs = layers.add([residual, shortcut]) + elif skip_connection_type == 'sum': + outputs = layers.add([residual, inputs]) + elif skip_connection_type == 'none': + outputs = residual + if return_skip: + return outputs, skip + else: + return outputs + +def Xception(inputs, alpha=1, downsample_factor=16): + if downsample_factor == 8: + entry_block3_stride = 1 + middle_block_rate = 2 # ! Not mentioned in paper, but required + exit_block_rates = (2, 4) + atrous_rates = (12, 24, 36) + elif downsample_factor == 16: + entry_block3_stride = 2 + middle_block_rate = 1 + exit_block_rates = (1, 2) + atrous_rates = (6, 12, 18) + else: + raise ValueError('Unsupported factor - `{}`, Use 8 or 16.'.format(downsample_factor)) + + # 256,256,32 + x = Conv2D(32, (3, 3), strides=(2, 2), + name='entry_flow_conv1_1', use_bias=False, padding='same')(inputs) + x = BatchNormalization(name='entry_flow_conv1_1_BN')(x) + x = Activation('relu')(x) + + # 256,256,64 + x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1) + x = BatchNormalization(name='entry_flow_conv1_2_BN')(x) + x = Activation('relu')(x) + + # 256,256,128 -> 256,256,128 -> 128,128,128 + x = _xception_block(x, [128, 128, 128], 'entry_flow_block1', + skip_connection_type='conv', stride=2, + depth_activation=False) + + # 128,128,256 -> 128,128,256 -> 64,64,256 + # skip = 128,128,256 + x, skip1 = _xception_block(x, [256, 256, 256], 'entry_flow_block2', + skip_connection_type='conv', stride=2, + depth_activation=False, return_skip=True) + + x = _xception_block(x, [728, 728, 728], 'entry_flow_block3', + skip_connection_type='conv', stride=entry_block3_stride, + depth_activation=False) + for i in range(16): + x = _xception_block(x, [728, 728, 728], 'middle_flow_unit_{}'.format(i + 1), + skip_connection_type='sum', stride=1, rate=middle_block_rate, + depth_activation=False) + + x = _xception_block(x, [728, 1024, 1024], 'exit_flow_block1', + skip_connection_type='conv', stride=1, rate=exit_block_rates[0], + depth_activation=False) + x = _xception_block(x, [1536, 1536, 2048], 'exit_flow_block2', + skip_connection_type='none', stride=1, rate=exit_block_rates[1], + depth_activation=True) + return x,atrous_rates,skip1 \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/deeplab.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/deeplab.py new file mode 100644 index 0000000..08b3f76 --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/deeplab.py @@ -0,0 +1,176 @@ +# /usr/bin/env python3 +# -*- mode: python -*- + +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +from __future__ import absolute_import, division, print_function + +import tensorflow as tf +from tensorflow.keras import backend as K +from tensorflow.keras.layers import (Activation, BatchNormalization, + Concatenate, Conv2D, DepthwiseConv2D, + Dropout, GlobalAveragePooling2D, Input, + Lambda, Softmax, ZeroPadding2D) +from tensorflow.keras.models import Model + +from aimet_zoo_tensorflow.deeplabv3plus_tf2.model.nets.Xception import Xception +from aimet_zoo_tensorflow.deeplabv3plus_tf2.model.nets.mobilenet import mobilenetV2 + +def SepConv_BN(x, filters, prefix, stride=1, kernel_size=3, rate=1, depth_activation=False, epsilon=1e-3): + # paddinghwǷҪ + if stride == 1: + depth_padding = 'same' + else: + kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) + pad_total = kernel_size_effective - 1 + pad_beg = pad_total // 2 + pad_end = pad_total - pad_beg + x = ZeroPadding2D((pad_beg, pad_end))(x) + depth_padding = 'valid' + + # Ҫ + if not depth_activation: + x = Activation('relu')(x) + + # 3x31x1 + # 3x3; + x = DepthwiseConv2D((kernel_size, kernel_size), strides=(stride, stride), dilation_rate=(rate, rate), + padding=depth_padding, use_bias=False, name=prefix + '_depthwise')(x) + x = BatchNormalization(name=prefix + '_depthwise_BN', epsilon=epsilon)(x) + if depth_activation: + x = Activation('relu')(x) + + # 1x1ѹ + x = Conv2D(filters, (1, 1), padding='same', + use_bias=False, name=prefix + '_pointwise')(x) + x = BatchNormalization(name=prefix + '_pointwise_BN', epsilon=epsilon)(x) + if depth_activation: + x = Activation('relu')(x) + + return x + +def Deeplabv3(input_shape, num_classes, alpha=1., backbone="mobilenet", downsample_factor=16): + img_input = Input(shape=input_shape) + + if backbone=="xception": + #----------------------------------# + # + # dzskip1 [128,128,256] + # ɲx [30,30,2048] + #----------------------------------# + x, atrous_rates, skip1 = Xception(img_input, alpha, downsample_factor=downsample_factor) + elif backbone=="mobilenet": + #----------------------------------# + # + # dzskip1 [128,128,24] + # ɲx [30,30,320] + #----------------------------------# + x, atrous_rates, skip1 = mobilenetV2(img_input, alpha, downsample_factor=downsample_factor) + else: + raise ValueError('Unsupported backbone - `{}`, Use mobilenet, xception.'.format(backbone)) + + size_before = tf.keras.backend.int_shape(x) + + #-----------------------------------------# + # һ֧ + # ASPPȡģ + # òͬʵ;ȡ + #-----------------------------------------# + # ֧0 + b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0')(x) + b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0) + b0 = Activation('relu', name='aspp0_activation')(b0) + + # ֧1 rate = 6 (12) + b1 = SepConv_BN(x, 256, 'aspp1', + rate=atrous_rates[0], depth_activation=True, epsilon=1e-5) + # ֧2 rate = 12 (24) + b2 = SepConv_BN(x, 256, 'aspp2', + rate=atrous_rates[1], depth_activation=True, epsilon=1e-5) + # ֧3 rate = 18 (36) + b3 = SepConv_BN(x, 256, 'aspp3', + rate=atrous_rates[2], depth_activation=True, epsilon=1e-5) + + # ֧4 ȫƽexpand_dimsάȣ֮1x1ͨ + b4 = GlobalAveragePooling2D()(x) + b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4) + b4 = Lambda(lambda x: K.expand_dims(x, 1))(b4) + b4 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='image_pooling')(b4) + b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4) + b4 = Activation('relu')(b4) + # ֱresize_imageshw + b4 = Lambda(lambda x: tf.compat.v1.image.resize_images(x, size_before[1:3], align_corners=True))(b4) + + #-----------------------------------------# + # ֧ݶѵ + # Ȼ1x1 + #-----------------------------------------# + x = Concatenate()([b4, b0, b1, b2, b3]) + # conv2dѹ 32,32,256 + x = Conv2D(256, (1, 1), padding='same', use_bias=False, name='concat_projection')(x) + x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x) + x = Activation('relu')(x) + x = Dropout(0.1)(x) + + skip_size = tf.keras.backend.int_shape(skip1) + #-----------------------------------------# + # ǿϲ + #-----------------------------------------# + x = Lambda(lambda xx: tf.compat.v1.image.resize_images(xx, skip_size[1:3], align_corners=True))(x) + #----------------------------------# + # dz + #----------------------------------# + dec_skip1 = Conv2D(48, (1, 1), padding='same',use_bias=False, name='feature_projection0')(skip1) + dec_skip1 = BatchNormalization(name='feature_projection0_BN', epsilon=1e-5)(dec_skip1) + dec_skip1 = Activation(tf.nn.relu)(dec_skip1) + + #-----------------------------------------# + # dzѵþȡ + #-----------------------------------------# + x = Concatenate()([x, dec_skip1]) + x = SepConv_BN(x, 256, 'decoder_conv0', + depth_activation=True, epsilon=1e-5) + x = SepConv_BN(x, 256, 'decoder_conv1', + depth_activation=True, epsilon=1e-5) + + #-----------------------------------------# + # ÿصķ + #-----------------------------------------# + # 512,512 + size_before3 = tf.keras.backend.int_shape(img_input) + # 512,512,21 + x = Conv2D(num_classes, (1, 1), padding='same')(x) + x = Lambda(lambda xx:tf.compat.v1.image.resize_images(xx,size_before3[1:3], align_corners=True))(x) + x = Softmax()(x) + + model = Model(img_input, x, name='deeplabv3plus') + return model \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/mobilenet.py b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/mobilenet.py new file mode 100644 index 0000000..193bac1 --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/model/nets/mobilenet.py @@ -0,0 +1,169 @@ +# /usr/bin/env python3 +# -*- mode: python -*- + +# MIT License + +# Copyright (c) 2021 Bubbliiiing + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +from tensorflow.keras.activations import relu +from tensorflow.keras.layers import (Activation, Add, BatchNormalization, + Conv2D, DepthwiseConv2D) + + +def _make_divisible(v, divisor, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + if new_v < 0.9 * v: + new_v += divisor + return new_v + +def relu6(x): + return relu(x, max_value=6) + +def _inverted_res_block(inputs, expansion, stride, alpha, in_filters, filters, block_id, skip_connection, rate=1): + pointwise_filters = _make_divisible(int(filters * alpha), 8) + prefix = 'expanded_conv_{}_'.format(block_id) + + x = inputs + #----------------------------------------------------# + # 1x1ͨͨ + #----------------------------------------------------# + if block_id: + x = Conv2D(expansion * in_filters, kernel_size=1, padding='same', + use_bias=False, activation=None, + name=prefix + 'expand')(x) + x = BatchNormalization(epsilon=1e-3, momentum=0.999, + name=prefix + 'expand_BN')(x) + x = Activation(relu6, name=prefix + 'expand_relu')(x) + else: + prefix = 'expanded_conv_' + + #----------------------------------------------------# + # ȿɷȡ + #----------------------------------------------------# + x = DepthwiseConv2D(kernel_size=3, strides=stride, activation=None, + use_bias=False, padding='same', dilation_rate=(rate, rate), + name=prefix + 'depthwise')(x) + x = BatchNormalization(epsilon=1e-3, momentum=0.999, + name=prefix + 'depthwise_BN')(x) + + x = Activation(relu6, name=prefix + 'depthwise_relu')(x) + + #----------------------------------------------------# + # 1x1ľͨ½ + #----------------------------------------------------# + x = Conv2D(pointwise_filters, + kernel_size=1, padding='same', use_bias=False, activation=None, + name=prefix + 'project')(x) + x = BatchNormalization(epsilon=1e-3, momentum=0.999, + name=prefix + 'project_BN')(x) + + #----------------------------------------------------# + # Ӳв + #----------------------------------------------------# + if skip_connection: + return Add(name=prefix + 'add')([inputs, x]) + return x + +def mobilenetV2(inputs, alpha=1, downsample_factor=8): + if downsample_factor == 8: + block4_dilation = 2 + block5_dilation = 4 + block4_stride = 1 + atrous_rates = (12, 24, 36) + elif downsample_factor == 16: + block4_dilation = 1 + block5_dilation = 2 + block4_stride = 2 + atrous_rates = (6, 12, 18) + else: + raise ValueError('Unsupported factor - `{}`, Use 8 or 16.'.format(downsample_factor)) + + first_block_filters = _make_divisible(32 * alpha, 8) + # 512,512,3 -> 256,256,32 + x = Conv2D(first_block_filters, + kernel_size=3, + strides=(2, 2), padding='same', + use_bias=False, name='Conv')(inputs) + x = BatchNormalization( + epsilon=1e-3, momentum=0.999, name='Conv_BN')(x) + x = Activation(relu6, name='Conv_Relu6')(x) + + + x = _inverted_res_block(x, in_filters=32, filters=16, alpha=alpha, stride=1, + expansion=1, block_id=0, skip_connection=False) + + #---------------------------------------------------------------# + # 256,256,16 -> 128,128,24 + x = _inverted_res_block(x, in_filters=16, filters=24, alpha=alpha, stride=2, + expansion=6, block_id=1, skip_connection=False) + x = _inverted_res_block(x, in_filters=24, filters=24, alpha=alpha, stride=1, + expansion=6, block_id=2, skip_connection=True) + skip1 = x + #---------------------------------------------------------------# + # 128,128,24 -> 64,64.32 + x = _inverted_res_block(x, in_filters=24, filters=32, alpha=alpha, stride=2, + expansion=6, block_id=3, skip_connection=False) + x = _inverted_res_block(x, in_filters=32, filters=32, alpha=alpha, stride=1, + expansion=6, block_id=4, skip_connection=True) + x = _inverted_res_block(x, in_filters=32, filters=32, alpha=alpha, stride=1, + expansion=6, block_id=5, skip_connection=True) + #---------------------------------------------------------------# + # 64,64,32 -> 32,32.64 + x = _inverted_res_block(x, in_filters=32, filters=64, alpha=alpha, stride=block4_stride, + expansion=6, block_id=6, skip_connection=False) + x = _inverted_res_block(x, in_filters=64, filters=64, alpha=alpha, stride=1, rate=block4_dilation, + expansion=6, block_id=7, skip_connection=True) + x = _inverted_res_block(x, in_filters=64, filters=64, alpha=alpha, stride=1, rate=block4_dilation, + expansion=6, block_id=8, skip_connection=True) + x = _inverted_res_block(x, in_filters=64, filters=64, alpha=alpha, stride=1, rate=block4_dilation, + expansion=6, block_id=9, skip_connection=True) + + # 32,32.64 -> 32,32.96 + x = _inverted_res_block(x, in_filters=64, filters=96, alpha=alpha, stride=1, rate=block4_dilation, + expansion=6, block_id=10, skip_connection=False) + x = _inverted_res_block(x, in_filters=96, filters=96, alpha=alpha, stride=1, rate=block4_dilation, + expansion=6, block_id=11, skip_connection=True) + x = _inverted_res_block(x, in_filters=96, filters=96, alpha=alpha, stride=1, rate=block4_dilation, + expansion=6, block_id=12, skip_connection=True) + + #---------------------------------------------------------------# + # 32,32.96 -> 32,32,160 -> 32,32,320 + x = _inverted_res_block(x, in_filters=96, filters=160, alpha=alpha, stride=1, rate=block4_dilation, # 1! + expansion=6, block_id=13, skip_connection=False) + x = _inverted_res_block(x, in_filters=160, filters=160, alpha=alpha, stride=1, rate=block5_dilation, + expansion=6, block_id=14, skip_connection=True) + x = _inverted_res_block(x, in_filters=160, filters=160, alpha=alpha, stride=1, rate=block5_dilation, + expansion=6, block_id=15, skip_connection=True) + + x = _inverted_res_block(x, in_filters=160, filters=320, alpha=alpha, stride=1, rate=block5_dilation, + expansion=6, block_id=16, skip_connection=False) + return x,atrous_rates,skip1 \ No newline at end of file diff --git a/aimet_zoo_tensorflow/deeplabv3plus_tf2/requirements.txt b/aimet_zoo_tensorflow/deeplabv3plus_tf2/requirements.txt new file mode 100644 index 0000000..67b512e --- /dev/null +++ b/aimet_zoo_tensorflow/deeplabv3plus_tf2/requirements.txt @@ -0,0 +1 @@ +matplotlib==3.2.1 diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/MobileNetEdgeTPU.md b/aimet_zoo_tensorflow/mobilenetedgetpu/MobileNetEdgeTPU.md new file mode 100755 index 0000000..2efb250 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenetedgetpu/MobileNetEdgeTPU.md @@ -0,0 +1,55 @@ +# MobileNet-EdgeTPU + +## Environment Setup + +### Setup AI Model Efficiency Toolkit (AIMET) +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.26/packaging/install.md) before proceeding further. This evaluation was run using [AIMET 1.26.0 for TensorFlow 2.4](https://github.com/quic/aimet/releases/tag/1.26.0) i.e. please set `release_tag="1.26.0"` and `AIMET_VARIANT="tf_gpu"` in the above instructions. + +### Append the repo location to your `PYTHONPATH` by doing the following: + `export PYTHONPATH=$PYTHONPATH://aimet-model-zoo` + +## Dataset +- ImageNet can be downloaded from here: + - http://www.image-net.org/ +- For this evaluation, Tf-records of ImageNet validation dataset are required. (See https://github.com/tensorflow/models/tree/master/research/slim#Data for details) +- The Tf-records of ImageNet validation dataset should be organized in the following way +```bash +< path to ImageNet validation dataset Tf-records> +├── validation-00000-of-00128 +├── validation-00001-of-00128 +├── ... +``` + + +--- + +## Model checkpoint for AIMET optimization + - Downloading of model checkpoints is handled by evaluation script. + - Checkpoint used for AIMET quantization can be downloaded from the [Releases](/../../releases) page. + + --- + +## Usage +```bash +python aimet_zoo_tensorflow/mobilenetedgetpu/evaluators/mobilenet_edgetpu_quanteval.py + --model-config \ + --dataset-path + --batch-size +``` + +* example + ``` + python mobilenet_edgetpu_quanteval.py --dataset-path --model-config mobilenetedgetpu_w8a8 --batch-size 64 + ``` + +--- + +## Quantization configuration +In the evaluation script included, we have manually configured the quantizer ops with the following assumptions: + ++ Weight quantization: 8 bits, per-channel symmetric quantization ++ Bias parameters are not quantized ++ Activation quantization: 8 bits, asymmetric quantization ++ TF_enhanced was used for weight quantization scheme ++ TF_enhanced was used for activation quantization scheme ++ Batch_norm_fold was used for weight diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/__init__.py b/aimet_zoo_tensorflow/mobilenetedgetpu/__init__.py new file mode 100755 index 0000000..c1b6edb --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenetedgetpu/__init__.py @@ -0,0 +1,2 @@ +""" MobileNet Edge TPU """ +from .model.model_definition import MobileNet diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/dataloader/__init__.py b/aimet_zoo_tensorflow/mobilenetedgetpu/dataloader/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_tensorflow/mobilenetedgetpu/dataloader/dataloaders_and_eval_func.py new file mode 100755 index 0000000..5ebf007 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenetedgetpu/dataloader/dataloaders_and_eval_func.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +#pylint: skip-file +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +# ============================================================================== +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import os +from glob import glob +import numpy as np +from tqdm import tqdm +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" +import logging + +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + +assert tf.__version__ >= "2" +logger = logging.getLogger(__file__) + + +def load_graph(graph, meta_graph, checkpoint=None): + """ + Load a TF graph given the meta and checkpoint files + :param graph: Graph to load into + :param meta_graph: Meta file + :param checkpoint: Checkpoint file + :return: Newly created TF session + """ + gpu_options = tf.GPUOptions(allow_growth=True) + config = tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options) + sess = tf.Session(config=config, graph=graph) + # Open the graph and restore the parameters + saver = tf.train.import_meta_graph(meta_graph, clear_devices=True) + if checkpoint is None: + checkpoint = meta_graph.split('.meta')[0] + saver.restore(sess, checkpoint) + return sess, saver + + +def initialize_uninitialized_vars(sess): + """ + Some graphs have variables created after training that need to be initialized. + However, in pre-trained graphs we don't want to reinitialize variables that are already + which would overwrite the values obtained during training. Therefore search for all + uninitialized variables and initialize ONLY those variables. + :param sess: TF session + :return: + """ + from itertools import compress + global_vars = tf.global_variables() + is_not_initialized = sess.run([~(tf.is_variable_initialized(var)) for var in global_vars]) + uninitialized_vars = list(compress(global_vars, is_not_initialized)) + if uninitialized_vars: + sess.run(tf.variables_initializer(uninitialized_vars)) + + +class ImagenetParser: + """ Parses ImageNet dataset """ + + def __init__(self, data_inputs=None, validation_inputs=None, batch_size=1): + """ + Constructor + :param data_inputs: List of input ops for the model + :param validation_inputs: List of validation ops for the model + :param batch_size: Batch size for the data + """ + + if not data_inputs: + data_inputs = ['data'] + + if len(data_inputs) > 1: + raise ValueError("Only one data input supported for imagenet") + self._data_inputs = data_inputs + + if not validation_inputs: + validation_inputs = ['labels'] + + if len(validation_inputs) > 1: + raise ValueError("Only one validation input supported for imagenet") + self._validation_inputs = validation_inputs + self._batch_size = batch_size + + @staticmethod + def parse(serialized_example): + """ + Parse one example + :param serialized_example: + :return: Input image and labels + """ + dim = 224 + + features = tf.compat.v1.parse_single_example(serialized_example, + features={ + 'image/class/label': tf.FixedLenFeature([], tf.int64), + 'image/encoded': tf.FixedLenFeature([], tf.string)}) + + image_data = features["image/encoded"] + image = tf.image.decode_jpeg(image_data, channels=3) + label = tf.cast(features["image/class/label"], tf.int32) + labels = tf.one_hot(indices=label, depth=1001) + with tf.compat.v1.name_scope(None,'eval_image', [image]): + if image.dtype != tf.float32: + image = tf.image.convert_image_dtype(image, dtype=tf.float32) + # Crop the central region of the image with an area containing 87.5% of + # the original image. + image = tf.image.central_crop(image, central_fraction=0.875) + + # Resize the image to the specified height and width. + image = tf.expand_dims(image, 0) + image = tf.image.resize_bilinear(image, [224, 224], align_corners=False) + image = tf.squeeze(image, [0]) + image = tf.subtract(image, 0.5) + image = tf.multiply(image, 2.0) + + return image, labels + + def get_batch(self, iterator, sess): + """ + Get the next batch of data + :param iterator: Data iterator + :return: Input images and labels in feed_dict form + """ + data, labels = iterator.get_next() + np_images, np_labels = sess.run([data, labels]) + return {self._data_inputs[0]: np_images, self._validation_inputs[0]: np_labels} + + def get_batch_size(self): + """ + Returns the batch size + :return: + """ + return self._batch_size + + def get_data_inputs(self): + """ + Get a list of data input + :return: List of data input ops + """ + return self._data_inputs + + def get_validation_inputs(self): + """ + Get a list of validation input + :return: List of validation input ops + """ + return self._validation_inputs + + +class TfRecordGenerator: + + """ Dataset generator for TfRecords""" + + def __init__(self, tfrecords, parser=ImagenetParser(), num_gpus=1, num_epochs=None): + """ + Constructor + :param tfrecords: A list of TfRecord files + :param parser: Defaults to use the ImageNet tfrecord parser, but any custom + parser function can be passed to read a custom tfrecords format. + :param num_gpus: The number of GPUs being used. Data batches must be generated for each GPU device + :param num_epochs: How many times to repeat the dataset. Default is forever. Then the + amount of data generated is determined by the number of iterations the model is run and the batch + size. If set to a specific number the dataset will only provide the amount of the total dataset + 'num_epochs' times. + :return: A new TfRecord generator used to generate data for model analysis + """ + + self._parser = parser + self._num_gpus = num_gpus + + # Setup the Dataset reader + self._dataset = tf.data.TFRecordDataset(tfrecords).repeat(num_epochs) + batch_size = parser.get_batch_size() + self._dataset = self._dataset.map(parser.parse, num_parallel_calls=1) + self._dataset = self._dataset.batch(batch_size) + + # Initialize the iterator. This must be allocated during init when the + # generator is to be used manually. Otherwise the generator will generate a + # new iterator each time it's used as an iterator + self._iterator = tf.compat.v1.data.make_one_shot_iterator(self._dataset) + + def __iter__(self): + """ + Iter method for the generator + :return: + """ + # creating one shot iterator ops in same graph as dataset ops + # TODO: this will keep adding iterator ops in the same graph every time this iter method is being called, need + # better solution + + # pylint: disable=protected-access + with self._dataset._graph.as_default(): + self._iterator = tf.compat.v1.data.make_one_shot_iterator(self._dataset) + self.sess = tf.Session() + return self + + def __next__(self): + """ + Return the next set of batched data + **NOTE** This function will not return new batches until the previous batches have + actually been used by a call to tensorflow. Eg used in a graph with a call to + 'run' etc. If it's unused the same tensors will be returned over and over again. + :return: + """ + return self._parser.get_batch(self._iterator, self.sess) + + # Map next for python27 compatibility + next = __next__ + + def get_data_inputs(self): + """ + Returns a list of data input ops + :return: + """ + return self._parser.get_data_inputs() + + def get_validation_inputs(self): + """ + Returns a list of validation input ops + :return: + """ + return self._parser.get_validation_inputs() + + +class Dataloader: + """ Fetches the TFRecords of images, and supports running them through a TF Session""" + def __init__(self, generator): + self._generator = generator + + def run_graph(self, session, iterations): + """ + Evaluates the graph's performance by running data through the network + and calling an evaluation function to generate the performance metric. + :param session: The tensorflow session that contains the graph + :param iterations: The number of iterations (batches) to run through the network + :return: + """ + initialize_uninitialized_vars(session) + image_tensor = session.graph.get_tensor_by_name('MobilenetEdgeTPU/input:0') + eval_outputs = session.graph.get_tensor_by_name('MobilenetEdgeTPU/Predictions/Reshape_1:0') + + + counters = {'skipped': 0, 'success': 0} + cnt, count = 0, 0 + results_dict = {} + bar = tqdm(zip(range(iterations), self._generator)) + try: + for _, input_dict in bar: + # Setup the feed dictionary + images = input_dict['data'] + labels = input_dict['labels'] + + try: + output_data = session.run(eval_outputs, feed_dict={image_tensor:images}) + indices = np.argmax(output_data, axis=1) + labels = np.argmax(labels, axis=1) + + cnt += np.sum(indices==labels) + count += len(indices) + counters['success'] += 1 + bar.set_description(f'total count of samples: {count}, correctly predicted samples: {cnt}') + except tf.errors.InvalidArgumentError: + counters['skipped'] += 1 + except tf.errors.OutOfRangeError: + logger.info("Completed evaluation iterations: %i, success: %i, skipped: %i", + iterations, counters['success'], counters['skipped']) + finally: + acc_top1 = cnt/count + results_dict["acc_top1"] = acc_top1 + + return results_dict + + def forward_func(self, sess, callback_args: dict): + """ forward pass to compute encodings with """ + return self.run_graph(sess, iterations=callback_args['iterations']) + + +def get_dataloader(dataset_dir, batch_size): + """returns a Dataloader object for evaluation""" + parser = ImagenetParser(batch_size=batch_size) + tf_records = glob(os.path.join(dataset_dir, "validation*")) + generator = TfRecordGenerator( + tfrecords=tf_records, + parser=parser, + num_epochs=1) + dataloader = Dataloader(generator=generator) + return dataloader diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/evaluators/__init__.py b/aimet_zoo_tensorflow/mobilenetedgetpu/evaluators/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/evaluators/mobilenet_edgetpu_quanteval.py b/aimet_zoo_tensorflow/mobilenetedgetpu/evaluators/mobilenet_edgetpu_quanteval.py new file mode 100755 index 0000000..ab0435c --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenetedgetpu/evaluators/mobilenet_edgetpu_quanteval.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# pylint: disable=E0401,E1101,W0621,R0915,R0914,R0912 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" AIMET Quantsim evaluation code for MobileNetEdgeTPU """ + +import logging +import argparse +import os +import tensorflow.compat.v1 as tf +from tensorflow.compat.v1 import ConfigProto +from tensorflow.compat.v1 import InteractiveSession +from aimet_zoo_tensorflow.mobilenetedgetpu.model.model_definition import MobileNet +from aimet_zoo_tensorflow.mobilenetedgetpu.dataloader.dataloaders_and_eval_func import get_dataloader + +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" +tf.disable_v2_behavior() +assert tf.__version__ >= "2" +logger = logging.getLogger(__file__) + +config = ConfigProto() +config.gpu_options.allow_growth = True +session = InteractiveSession(config=config) + + +def arguments(raw_args): + """argument parser""" + parser = argparse.ArgumentParser( + description="Evaluation script for TensorFlow MobileNet-EdgeTPU." + ) + parser.add_argument( + "--model-config", + help="Model configuration to evaluate", + default="mobilenetedgetpu_w8a8", + choices=["mobilenetedgetpu_w8a8"], + ) + parser.add_argument( + "--dataset-path", help="Dir path to dataset in TFRecord format", required=True + ) + parser.add_argument( + "--batch-size", help="Data batch size for a model", type=int, default=32 + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU", type=bool, default=True + ) + args = parser.parse_args(raw_args) + return args + + +class ModelConfig: + """Hardcoded model configuration""" + + def __init__(self, args): + args.eval_num_examples = 50000 + for arg in vars(args): + setattr(self, arg, getattr(args, arg)) + + +def main(raw_args=None): + """Evaluation main function""" + args = arguments(raw_args) + config = ModelConfig(args) + + dataloader = get_dataloader( + dataset_dir=config.dataset_path, + batch_size=config.batch_size + ) + + model = MobileNet(model_config=config.model_config) + float_sess = model.get_session(quantized=False) + iterations = int(config.eval_num_examples / config.batch_size) + iterations_calibration = int(2000 / config.batch_size) + + # Evaluate original performance + fp32_results_dict = dataloader.run_graph(session=float_sess, iterations=iterations) + print(f'FP32 top1 accuracy: {fp32_results_dict["acc_top1"]:0.3f}') + + # Compute activation encodings (only adaround param encodings are preloaded) + sim = model.get_quantsim(quantized=True) + sim.compute_encodings(dataloader.forward_func,forward_pass_callback_args={"iterations": iterations_calibration}) + + # Evaluate simulated quantization performance + int8_results_dict = dataloader.run_graph(session=sim.session, iterations=iterations) + print(f'Quantized top1 accuracy: {int8_results_dict["acc_top1"]:0.3f}') + + float_sess.close() + + +if __name__ == "__main__": + main() diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/model/__init__.py b/aimet_zoo_tensorflow/mobilenetedgetpu/model/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/model/model_cards/__init__.py b/aimet_zoo_tensorflow/mobilenetedgetpu/model/model_cards/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/model/model_cards/mobilenetedgetpu_w8a8.json b/aimet_zoo_tensorflow/mobilenetedgetpu/model/model_cards/mobilenetedgetpu_w8a8.json new file mode 100755 index 0000000..6bc52ba --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenetedgetpu/model/model_cards/mobilenetedgetpu_w8a8.json @@ -0,0 +1,30 @@ +{ + "name": "MobileNet Edge TPU", + "framework": "tensorflow", + "task": "classification", + "model_args": { + "num_classes": 1001, + "starting_op_names": ["IteratorGetNext"], + "output_op_names": ["MobilenetEdgeTPU/Logits/output"] + }, + "input_shape": [null, 224, 224, 3], + "trainig_dataset": "ImageNet", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "techniques": ["bn_fold"] + } + }, + "artifacts": { + "url_pre_opt_weights": null, + "url_post_opt_weights": null, + "url_adaround_encodings": null, + "url_aimet_encodings": null, + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json", + "url_zipped_checkpoint": "https://github.com/quic/aimet-model-zoo/releases/download/mobilenetedgetpu_tf1/mobileNetEdgeTPU_ckpt.zip" + } +} diff --git a/aimet_zoo_tensorflow/mobilenetedgetpu/model/model_definition.py b/aimet_zoo_tensorflow/mobilenetedgetpu/model/model_definition.py new file mode 100755 index 0000000..ec1eee9 --- /dev/null +++ b/aimet_zoo_tensorflow/mobilenetedgetpu/model/model_definition.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +"""Class for downloading and setting up of optmized and original mobilenetedgetpu model for AIMET model zoo""" +import os +import json +import pathlib +import tensorflow.compat.v1 as tf + +from aimet_tensorflow.quantsim import QuantizationSimModel # pylint: disable=import-error +from aimet_tensorflow.batch_norm_fold import fold_all_batch_norms # pylint: disable=import-error +from aimet_zoo_tensorflow.common.downloader import Downloader +tf.disable_v2_behavior() + + +class MobileNet(Downloader): + """Wrapper class for loading Tensorflow MobileNet""" + + def __init__(self, model_config=None): + parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.cfg = False + if model_config: + config_filepath = os.path.join(parent_dir, "model_cards/", model_config + ".json") + with open(config_filepath, encoding='UTF-8') as f_in: + self.cfg = json.load(f_in) + if model_config: + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_adaround_encodings=self.cfg["artifacts"]["url_adaround_encodings"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + url_zipped_checkpoint=self.cfg["artifacts"]["url_zipped_checkpoint"], + model_dir=parent_dir, + model_config=model_config, + ) + self.input_shape = tuple( + x if x is not None else 1 for x in self.cfg["input_shape"] + ) + self.starting_op_names = self.cfg["model_args"]["starting_op_names"] + self.output_op_names = self.cfg["model_args"]["output_op_names"] + + @classmethod + def from_pretrained(cls, quantized=False): + #pylint:disable = unused-argument + """load pretrained model + for tensorflow models, get_session is used instead + """ + return "For TF 1.X based models, use get_session()" + + def get_quantsim(self, quantized=False): + """return a QuantizationSimulation for the model""" + if quantized: + sess = self.get_session(quantized=True) + else: + sess = self.get_session(quantized=False) + quant_config = self.cfg["optimization_config"]["quantization_configuration"] + kwargs = { + "quant_scheme": quant_config["quant_scheme"], + "default_param_bw": quant_config["param_bw"], + "default_output_bw": quant_config["output_bw"], + "config_file": self.path_aimet_config, + "starting_op_names": self.starting_op_names, + "output_op_names": self.output_op_names, + "use_cuda": True, + } + sim = QuantizationSimModel(sess, **kwargs) + if self.path_aimet_encodings and quantized: + sim.load_encodings_to_sim(self.path_aimet_encodings) + if self.path_adaround_encodings and quantized: + sim.set_and_freeze_param_encodings(self.path_adaround_encodings) + return sim + + def get_session(self, quantized=False): + """return a pretrained session""" + if not self.cfg: + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) + self._download_pre_opt_weights() + self._download_post_opt_weights() + self._download_aimet_config() + self._download_aimet_encodings() + self._download_adaround_encodings() + self._download_compressed_checkpoint() + meta_graph = None + #pylint:disable = unused-variable + for root, dirs, files in os.walk(self.extract_dir): + for file in files: + if file.endswith(".meta"): + meta_graph = root + "/" + file + if not meta_graph: + return FileNotFoundError("meta file not found in checkpoint directory") + g = tf.Graph() + with g.as_default(): + saver = tf.train.import_meta_graph(meta_graph, clear_devices=True) + sess = tf.Session( + config=tf.ConfigProto( + allow_soft_placement=True, + gpu_options=tf.GPUOptions(allow_growth=True), + ), + graph=g, + ) + checkpoint = meta_graph.split(".meta")[0] + saver.restore(sess, checkpoint) + if quantized: + sess, folded_pairs = fold_all_batch_norms( + sess, self.starting_op_names, self.output_op_names + ) + return sess diff --git a/aimet_zoo_torch/bert/evaluators/bert_quanteval.py b/aimet_zoo_torch/bert/evaluators/bert_quanteval.py index f4a5819..dce091d 100644 --- a/aimet_zoo_torch/bert/evaluators/bert_quanteval.py +++ b/aimet_zoo_torch/bert/evaluators/bert_quanteval.py @@ -55,7 +55,7 @@ def parse_args(raw_args): default=None, required=True, help="Output directory", - ) + ) args = parser.parse_args(raw_args) for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) diff --git a/aimet_zoo_torch/bert/model/model_definition.py b/aimet_zoo_torch/bert/model/model_definition.py index 88d0010..dc21740 100644 --- a/aimet_zoo_torch/bert/model/model_definition.py +++ b/aimet_zoo_torch/bert/model/model_definition.py @@ -77,7 +77,7 @@ def __init__(self, model_config=None, args=None): self.training_args = training_args self.aux_args = aux_args self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) - self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "bert_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] @@ -88,10 +88,10 @@ def __init__(self, model_config=None, args=None): self.training_args.do_eval = True # setup the download path from arguments self.path_pre_opt_weights = self.aux_args.fmodel_path - + self.path_post_opt_weights = self.aux_args.qmodel_path - - + + def from_pretrained(self): """get original or optimized model Parameters: diff --git a/aimet_zoo_torch/common/downloader.py b/aimet_zoo_torch/common/downloader.py index 8a6a70f..7c09f70 100644 --- a/aimet_zoo_torch/common/downloader.py +++ b/aimet_zoo_torch/common/downloader.py @@ -132,13 +132,13 @@ def _convert_src_to_asset_url(self, src: str): """convert src url to asset url """ # 0. get release_tag and file_name from url - release_tag, file_name = self._find_tag(src) + release_tag, file_name = self._find_tag(src) # 1. read all release in to all_releases headers = { 'Authorization': 'token ' + self.GITHUB_TOKEN , 'Accept': 'application/json', } - + resp = requests.get(self.INTERNAL_REPO_URL,headers = headers,timeout=(4, 30)) all_releases = resp.json() @@ -172,12 +172,12 @@ def _download_from_internal(self, src: str, dst: str): if self.GITHUB_TOKEN is None: raise NameError("GITHUB_TOKEN not setup, not able to download from internal github url, exit program!") if self.INTERNAL_REPO_URL is None: - raise NameError("variable INTERNAL_REPO_URL not setup, use export INTERNAL_REPO_URL= to setup before continuing") + raise NameError("variable INTERNAL_REPO_URL not setup, use export INTERNAL_REPO_URL= to setup before continuing") asset_url = self._convert_src_to_asset_url(src) headers = { 'Authorization': 'token ' + self.GITHUB_TOKEN , 'Accept': 'application/octet-stream', - } + } resp = requests.get(asset_url,headers = headers, timeout=(4, 30) ) with open(dst, 'wb') as file: file.write(resp.content) diff --git a/aimet_zoo_torch/deeplabv3/evaluators/deeplabv3_quanteval.py b/aimet_zoo_torch/deeplabv3/evaluators/deeplabv3_quanteval.py index 5f31356..6ef1818 100644 --- a/aimet_zoo_torch/deeplabv3/evaluators/deeplabv3_quanteval.py +++ b/aimet_zoo_torch/deeplabv3/evaluators/deeplabv3_quanteval.py @@ -20,7 +20,7 @@ -def arguments(): +def arguments(raw_args=None): """ argument parser""" parser = argparse.ArgumentParser( description="Evaluation script for PyTorch ImageNet networks." @@ -53,7 +53,7 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -76,11 +76,11 @@ def get_num_classes(val_loader, device): num_classes = uniques.shape[0] - 1 return num_classes - -def main(): +#pylint:disable = too-many-locals +def main(raw_args=None): """ main evaluation function""" seed(0) - args = arguments() + args = arguments(raw_args) device = get_device(args) iterations = -1 print(f"device: {device}") @@ -136,6 +136,11 @@ def main(): f"Optimized Model | {args.default_param_bw}-bit Environment | mIoU: {mIoU_optim_int8:.4f}" ) + return {'mIoU_orig_fp32': mIoU_orig_fp32, + 'mIoU_orig_int8': mIoU_orig_int8, + 'mIoU_optim_fp32': mIoU_optim_fp32, + 'mIoU_optim_int8': mIoU_optim_int8} + if __name__ == "__main__": main() diff --git a/aimet_zoo_torch/distilbert/model/model_definition.py b/aimet_zoo_torch/distilbert/model/model_definition.py index f25ee73..00fed72 100644 --- a/aimet_zoo_torch/distilbert/model/model_definition.py +++ b/aimet_zoo_torch/distilbert/model/model_definition.py @@ -78,7 +78,7 @@ def __init__(self, model_config=None, args=None): self.training_args = training_args self.aux_args = aux_args self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) - self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "distilbert_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] diff --git a/aimet_zoo_torch/ffnet/evaluators/ffnet_quanteval.py b/aimet_zoo_torch/ffnet/evaluators/ffnet_quanteval.py index 4dc2e87..43861d9 100755 --- a/aimet_zoo_torch/ffnet/evaluators/ffnet_quanteval.py +++ b/aimet_zoo_torch/ffnet/evaluators/ffnet_quanteval.py @@ -77,7 +77,7 @@ def forward_pass(device, model, data_loader): _pred = model(inputs.to(device)) -def arguments(): +def arguments(raw_args=None): """ argument parser""" #pylint: disable=redefined-outer-name parser = argparse.ArgumentParser( @@ -120,7 +120,7 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -137,10 +137,11 @@ def __init__(self, args): setattr(self, arg, getattr(args, arg)) -def main(args): +def main(raw_args=None): """ main evaluation function""" # pylint: disable=redefined-outer-name, too-many-locals, no-member seed(1234) + args = arguments(raw_args) config = ModelConfig(args) device = get_device(args) print(f"device: {device}") @@ -213,7 +214,11 @@ def main(args): f"Optimized Model | {config.default_param_bw}-bit Environment | mIoU: {mIoU_optim_int8:.4f}" ) + return {'mIoU_orig_fp32': mIoU_orig_fp32, + 'mIoU_orig_int8': mIoU_orig_int8, + 'mIoU_optim_fp32': mIoU_optim_fp32, + 'mIoU_optim_int8': mIoU_optim_int8} + if __name__ == "__main__": - args = arguments() - main(args) + main() diff --git a/aimet_zoo_torch/inverseform/evaluators/inverseform_quanteval.py b/aimet_zoo_torch/inverseform/evaluators/inverseform_quanteval.py index f8156ab..a05f712 100644 --- a/aimet_zoo_torch/inverseform/evaluators/inverseform_quanteval.py +++ b/aimet_zoo_torch/inverseform/evaluators/inverseform_quanteval.py @@ -22,7 +22,7 @@ from aimet_zoo_torch.inverseform.dataloader.helper import get_dataloaders_and_eval_func -def arguments(): +def arguments(raw_args=None): """argument parser""" parser = argparse.ArgumentParser( description="Evaluation script for PyTorch InverseForm models." @@ -58,7 +58,7 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -72,9 +72,9 @@ def seed(seednum, use_cuda): torch.cuda.manual_seed_all(seednum) -def main(): +def main(raw_args=None): """main evaluation function""" - args = arguments() + args = arguments(raw_args) seed(0, args.use_cuda) # Load model @@ -86,15 +86,18 @@ def main(): dataset_path=args.dataset_path ) + fp32_mIoU = eval_func(model.model.cuda()) + sim.compute_encodings( forward_pass_callback=eval_func, forward_pass_callback_args=-1 ) # Evaluate quantized model - mIoU = eval_func(sim.model) - print("Quantized mIoU : {:0.4f}".format(mIoU)) + quant_mIoU = eval_func(sim.model) + print(f"Original Model mIoU: {fp32_mIoU}, Quantized mIoU : {quant_mIoU:0.4f}") + + return {'original_mIoU': fp32_mIoU, 'quantized_mIoU': quant_mIoU} if __name__ == "__main__": - # fire.Fire(main) main() diff --git a/aimet_zoo_torch/minilm/model/model_definition.py b/aimet_zoo_torch/minilm/model/model_definition.py index e30b2a1..ffb22ad 100644 --- a/aimet_zoo_torch/minilm/model/model_definition.py +++ b/aimet_zoo_torch/minilm/model/model_definition.py @@ -52,7 +52,7 @@ def __init__(self, model_config=None,args=None): if model_config: config_filepath = os.path.join( self.parent_dir, "model_cards", model_config + ".json" - ) + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( @@ -79,7 +79,7 @@ def __init__(self, model_config=None,args=None): self.training_args = training_args self.aux_args = aux_args self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) - self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "minilm_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] diff --git a/aimet_zoo_torch/mmaction2/evaluators/metrics/__init__.py b/aimet_zoo_torch/mmaction2/evaluators/metrics/__init__.py index 62542f8..72858f5 100644 --- a/aimet_zoo_torch/mmaction2/evaluators/metrics/__init__.py +++ b/aimet_zoo_torch/mmaction2/evaluators/metrics/__init__.py @@ -7,4 +7,5 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +""" Init file to import Aimet Anet Metric """ from .anet_metric import AIMETANetMetric diff --git a/aimet_zoo_torch/mmaction2/evaluators/mmaction2_quanteval.py b/aimet_zoo_torch/mmaction2/evaluators/mmaction2_quanteval.py index c4be472..41902bd 100644 --- a/aimet_zoo_torch/mmaction2/evaluators/mmaction2_quanteval.py +++ b/aimet_zoo_torch/mmaction2/evaluators/mmaction2_quanteval.py @@ -12,6 +12,7 @@ import argparse import torch +#pylint:disable = import-error from mmengine.config import Config from mmengine.runner import Runner diff --git a/aimet_zoo_torch/mmaction2/model/__init__.py b/aimet_zoo_torch/mmaction2/model/__init__.py index 1f6c04f..e1a2d7a 100644 --- a/aimet_zoo_torch/mmaction2/model/__init__.py +++ b/aimet_zoo_torch/mmaction2/model/__init__.py @@ -7,4 +7,5 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +"""import bmn libraries""" from .bmn import * diff --git a/aimet_zoo_torch/mmaction2/model/base_model/__init__.py b/aimet_zoo_torch/mmaction2/model/base_model/__init__.py index 3110a2e..31ecd2e 100644 --- a/aimet_zoo_torch/mmaction2/model/base_model/__init__.py +++ b/aimet_zoo_torch/mmaction2/model/base_model/__init__.py @@ -7,4 +7,5 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +"""import basemodel""" from .base_model import BaseModel diff --git a/aimet_zoo_torch/mmaction2/model/model_definition.py b/aimet_zoo_torch/mmaction2/model/model_definition.py index 58c70c7..e8a675f 100644 --- a/aimet_zoo_torch/mmaction2/model/model_definition.py +++ b/aimet_zoo_torch/mmaction2/model/model_definition.py @@ -15,6 +15,7 @@ import pathlib import torch +#pylint:disable = import-error from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim from aimet_zoo_torch.common.downloader import Downloader diff --git a/aimet_zoo_torch/mmaction2/runner/__init__.py b/aimet_zoo_torch/mmaction2/runner/__init__.py index 8a836e4..e00d896 100644 --- a/aimet_zoo_torch/mmaction2/runner/__init__.py +++ b/aimet_zoo_torch/mmaction2/runner/__init__.py @@ -7,4 +7,5 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= +"""loading AIMET test loop""" from .loops import AIMETTestLoop diff --git a/aimet_zoo_torch/mobilebert/MobileBert.md b/aimet_zoo_torch/mobilebert/MobileBert.md old mode 100644 new mode 100755 index 46a36a7..3f7c698 --- a/aimet_zoo_torch/mobilebert/MobileBert.md +++ b/aimet_zoo_torch/mobilebert/MobileBert.md @@ -2,11 +2,11 @@ This document describes evaluation of optimized checkpoints for transformer models Mobilebert-uncased for NL Classification and Question Answering tasks. ## AIMET installation and setup -Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.23/packaging/install.md) (*Torch GPU* variant) before proceeding further. +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.27/packaging/install.md) (*Torch GPU* variant) before proceeding further. **NOTE** - All AIMET releases are available here: https://github.com/quic/aimet/releases -- This model has been tested using AIMET version *1.23.0* (i.e. set `release_tag="1.23.0"` in the above instructions). +- This model has been tested using AIMET version *1.27.0* (i.e. set `release_tag="1.27.0"` in the above instructions). - This model is compatible with the PyTorch GPU variant of AIMET (i.e. set `AIMET_VARIANT="torch_gpu"` in the above instructions). ## Additional Setup Dependencies @@ -14,6 +14,12 @@ Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aime pip install datasets==2.4.0 pip install transformers==4.11.3 ``` + +## Add AIMET Model Zoo to the PYTHONPATH +```bash +export PYTHONPATH=$PYTHONPATH: +``` + ## Model checkpoint - Original full precision checkpoints without downstream training were downloaded through hugging face - [Full precision model with downstream training weight files] are automatically downloaded using evaluation script @@ -37,24 +43,74 @@ python mobilebert_quanteval.py \ python mobilebert_quanteval.py --model_config mobilebert_w8a8_rte --per_device_eval_batch_size 4 --output_dir ./evaluation_result ``` -* supported values of model_config are "mobilebert_w8a8_rte","mobilebert_w8a8_stsb","mobilebert_w8a8_mrpc","mobilebert_w8a8_cola","mobilebert_w8a8_sst2","mobilebert_w8a8_qnli","mobilebert_w8a8_qqp","mobilebert_w8a8_mnli", "mobilebert_w8a8_squad" +* supported values of model_config are "mobilebert_w8a8_rte","mobilebert_w8a8_stsb","mobilebert_w8a8_mrpc","mobilebert_w8a8_cola","mobilebert_w8a8_sst2","mobilebert_w8a8_qnli","mobilebert_w8a8_qqp","mobilebert_w8a8_mnli", "mobilebert_w8a8_squad", "mobilebert_w4a8_rte","mobilebert_w4a8_stsb","mobilebert_w4a8_mrpc","mobilebert_w4a8_cola","mobilebert_w4a8_sst2","mobilebert_w4a8_qnli","mobilebert_w4a8_qqp","mobilebert_w4a8_mnli", "mobilebert_w4a8_squad" ## Quantization Configuration -The following configuration has been used for the above models for INT8 quantization: -- Weight quantization: 8 bits, symmetric quantization +The following configuration has been used for the above models for IN4/INT8 quantization: +- Weight quantization: 4/8 bits, symmetric quantization - Bias parameters are not quantized - Activation quantization: 8 bits, asymmetric quantization - Model inputs are quantized -- TF range learning was used as quantization scheme +- Different quantization scheme for different weight bithwidth and downstreaming tasks - Mask values of -6 was applied in attention layers - Quantization aware training (QAT) was used to obtain optimized quantized weights, detailed hyperparameters listed in [Yelysei Bondarenko, Markus Nagel, Tijmen Blankevoort, "Understanding and Overcoming the Challenges of Efficient Transformer Quantization", EMNLP 2021](https://arxiv.org/abs/2109.12948). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
QAT Configuration CoLA SST-2 MRPC STS-B QQP MNLI QNLI RTE
W8A8 per-tensor, tf per-channel, tf per-channel, tf per-channel, tf per-channel, tf per-tensor, tf_enhanced per-tensor, tf_enhanced per-channel, tf
W4A8 per-tensor, tf_enhanced per-channel, tf_enhanced per-channel, tf_enhanced per-channel, tf_enhanced per-tensor, tf_enhanced per-tensor, tf_enhanced per-tensor, tf_enhanced per-channel, tf_enhanced
+ + + + + + + + + + + + + +
QAT Configuration
W8A8 per-channel, tf
W4A8 per-channel, range_learning_with_tf_enhanced_init
+ ## Results Below are the results of the Pytorch transformer model MobileBert for GLUE dataset: - + @@ -67,26 +123,61 @@ Below are the results of the Pytorch transformer model MobileBert for GLUE datas - - - - - - - - - + + + + + + + + + - - - - - - + + + + + + + + + + + + + + + + - - + + + + + +
Configuration CoLA (corr) SST-2 (acc) MRPC (f1)
FP32 50.41 90.83 85.47 88.75 90.26 83.36 90.81 70.04 81.24 51.48 91.60 85.86 88.22 90.66 83.54 91.18 68.60 81.27
W8A8 49.34 89.79 88.50 88.46 88.60 83.82 52.51 91.63 90.81 88.19 90.80 83.46 91.12 68.95 82.18
W4A8 50.34 91.28 87.61 87.30 90.48 70.40 81.17 82.90 89.42 68.23 80.95
+ + + + + + + + + + + + + + + + + + + + +
EM F1
FP32 82.75 90.11
W8A8 81.96 89.41
W4A8 81.88 89.33
diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_cola.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_cola.json new file mode 100755 index 0000000..4fd1689 --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_cola.json @@ -0,0 +1,45 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "task_name":"cola", + "max_seq_length":128, + "overwrite_cache":false, + "pad_to_max_length":true, + "train_file":null, + "validation_file":null, + "test_file":null, + "max_eval_samples":null + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "use_fast_tokenizer":true, + "model_revision":"main", + "use_auth_token":false, + "attention_probs_dropout_prob":0.1 + }, + "aux_args":{ + "fmodel_path":"../model/weights/fp.pth", + "qmodel_path":"../model/weights/qat.ckpt" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/cola_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/cola_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_quantsim_config.json" + } +} diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_mnli.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_mnli.json new file mode 100755 index 0000000..8149429 --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_mnli.json @@ -0,0 +1,45 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "task_name":"mnli", + "max_seq_length":128, + "overwrite_cache":false, + "pad_to_max_length":true, + "train_file":null, + "validation_file":null, + "test_file":null, + "max_eval_samples":null + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "use_fast_tokenizer":true, + "model_revision":"main", + "use_auth_token":false, + "attention_probs_dropout_prob":0.1 + }, + "aux_args":{ + "fmodel_path":"../model/weights/fp.pth", + "qmodel_path":"../model/weights/qat.ckpt" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/mnli_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/mnli_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_quantsim_config.json" + } +} diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_mrpc.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_mrpc.json new file mode 100755 index 0000000..41a7eb2 --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_mrpc.json @@ -0,0 +1,45 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "task_name":"mrpc", + "max_seq_length":128, + "overwrite_cache":false, + "pad_to_max_length":true, + "train_file":null, + "validation_file":null, + "test_file":null, + "max_eval_samples":null + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "use_fast_tokenizer":true, + "model_revision":"main", + "use_auth_token":false, + "attention_probs_dropout_prob":0.1 + }, + "aux_args":{ + "fmodel_path":"../model/weights/fp.pth", + "qmodel_path":"../model/weights/qat.ckpt" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_per_channel_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/mrpc_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/mrpc_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" + } +} diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_qnli.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_qnli.json new file mode 100755 index 0000000..04a215f --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_qnli.json @@ -0,0 +1,45 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "task_name":"qnli", + "max_seq_length":128, + "overwrite_cache":false, + "pad_to_max_length":true, + "train_file":null, + "validation_file":null, + "test_file":null, + "max_eval_samples":null + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "use_fast_tokenizer":true, + "model_revision":"main", + "use_auth_token":false, + "attention_probs_dropout_prob":0.1 + }, + "aux_args":{ + "fmodel_path":"../model/weights/fp.pth", + "qmodel_path":"../model/weights/qat.ckpt" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/qnli_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/qnli_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_quantsim_config.json" + } +} diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_qqp.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_qqp.json new file mode 100755 index 0000000..2c8b779 --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_qqp.json @@ -0,0 +1,45 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "task_name":"qqp", + "max_seq_length":128, + "overwrite_cache":false, + "pad_to_max_length":true, + "train_file":null, + "validation_file":null, + "test_file":null, + "max_eval_samples":null + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "use_fast_tokenizer":true, + "model_revision":"main", + "use_auth_token":false, + "attention_probs_dropout_prob":0.1 + }, + "aux_args":{ + "fmodel_path":"../model/weights/fp.pth", + "qmodel_path":"../model/weights/qat.ckpt" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/qqp_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/qqp_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_quantsim_config.json" + } +} diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_rte.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_rte.json new file mode 100755 index 0000000..76a293b --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_rte.json @@ -0,0 +1,45 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "task_name":"rte", + "max_seq_length":128, + "overwrite_cache":false, + "pad_to_max_length":true, + "train_file":null, + "validation_file":null, + "test_file":null, + "max_eval_samples":null + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "use_fast_tokenizer":true, + "model_revision":"main", + "use_auth_token":false, + "attention_probs_dropout_prob":0.1 + }, + "aux_args":{ + "fmodel_path":"../model/weights/fp.pth", + "qmodel_path":"../model/weights/qat.ckpt" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_per_channel_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/rte_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/rte_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" + } +} diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_squad.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_squad.json new file mode 100755 index 0000000..caad017 --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_squad.json @@ -0,0 +1,53 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "dataset_name":"squad", + "dataset_config_name":null, + "train_file":null, + "validation_file":null, + "test_file":null, + "overwrite_cache":false, + "preprocessing_num_workers":null, + "max_seq_length":384, + "pad_to_max_length":true, + "max_train_samples":null, + "max_eval_samples":null, + "max_predict_samples":null, + "version_2_with_negative":false, + "null_score_diff_threshold":0.0, + "doc_stride":128, + "n_best_size":20, + "max_answer_length":30 + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "model_revision":"main", + "use_auth_token":false + }, + "aux_args":{ + "fmodel_path":"../model/weights/pre_opt_weights", + "qmodel_path":"../model/weights/post_opt_weights" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "range_learning_with_tf_enhanced_init", + "aimet_config": "htp_per_channel_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/squad_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/squad_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" + } +} + diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_sst2.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_sst2.json new file mode 100755 index 0000000..023ff2d --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_sst2.json @@ -0,0 +1,45 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "task_name":"sst2", + "max_seq_length":128, + "overwrite_cache":false, + "pad_to_max_length":true, + "train_file":null, + "validation_file":null, + "test_file":null, + "max_eval_samples":null + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "use_fast_tokenizer":true, + "model_revision":"main", + "use_auth_token":false, + "attention_probs_dropout_prob":0.1 + }, + "aux_args":{ + "fmodel_path":"../model/weights/fp.pth", + "qmodel_path":"../model/weights/qat.ckpt" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/sst2_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/sst2_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_quantsim_config.json" + } +} diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_stsb.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_stsb.json new file mode 100755 index 0000000..bbb914a --- /dev/null +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w4a8_stsb.json @@ -0,0 +1,45 @@ +{ + "name": "MOBILEBERT", + "framework": "pytorch", + "task": "", + "data_training_args":{ + "task_name":"stsb", + "max_seq_length":128, + "overwrite_cache":false, + "pad_to_max_length":true, + "train_file":null, + "validation_file":null, + "test_file":null, + "max_eval_samples":null + }, + "model_args": { + "model_name_or_path":"google/mobilebert-uncased", + "config_name":null, + "tokenizer_name":null, + "cache_dir":null, + "use_fast_tokenizer":true, + "model_revision":"main", + "use_auth_token":false, + "attention_probs_dropout_prob":0.1 + }, + "aux_args":{ + "fmodel_path":"../model/weights/fp.pth", + "qmodel_path":"../model/weights/qat.ckpt" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 4, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_per_channel_quantsim_config", + "techniques": ["qat"] + } + }, + "artifacts": { + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/stsb_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/stsb_w4a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" + } +} diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json old mode 100644 new mode 100755 index c3febc1..a369588 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json @@ -32,13 +32,14 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf", + "aimet_config": "htp_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/cola_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/cola_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/cola_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/cola_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json old mode 100644 new mode 100755 index 885178b..887fc33 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json @@ -32,13 +32,14 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/mnli_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/mnli_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/mnli_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/mnli_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json old mode 100644 new mode 100755 index 6931e3e..bdee3d7 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json @@ -32,13 +32,14 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf", + "aimet_config": "htp_per_channel_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/mrpc_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/mrpc_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/mrpc_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/mrpc_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json old mode 100644 new mode 100755 index f20e87c..0818c34 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json @@ -32,13 +32,14 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf_enhanced", + "aimet_config": "htp_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/qnli_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/qnli_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/qnli_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/qnli_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json old mode 100644 new mode 100755 index 1db746f..ada29e8 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json @@ -32,13 +32,14 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf", + "aimet_config": "htp_per_channel_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/qqp_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/qqp_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/qqp_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/qqp_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json old mode 100644 new mode 100755 index d224d75..1e267bd --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json @@ -32,13 +32,14 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf", + "aimet_config": "htp_per_channel_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/rte_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/rte_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/rte_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/rte_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json old mode 100644 new mode 100755 index 259b595..ab05916 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json @@ -39,14 +39,15 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf", + "aimet_config": "htp_per_channel_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/squad_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/squad_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/squad_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/squad_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json old mode 100644 new mode 100755 index cb02537..4c8a6c1 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json @@ -32,13 +32,14 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf", + "aimet_config": "htp_per_channel_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/sst2_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/sst2_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/sst2_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/sst2_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json old mode 100644 new mode 100755 index 18e3701..a08cf71 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json @@ -32,13 +32,14 @@ "param_bw": 8, "output_bw": 8, "input_quantization": true, - "quant_scheme": "tf_range_learning", + "quant_scheme": "tf", + "aimet_config": "htp_per_channel_quantsim_config", "techniques": ["qat"] } }, "artifacts": { - "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/stsb_fp.pth", - "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/torch_mobilebert/stsb_qat.ckpt", - "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + "url_pre_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/stsb_fp.pth", + "url_post_opt_weights":"https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/stsb_w8a8_qat.ckpt", + "url_aimet_config": "https://github.com/quic/aimet-model-zoo/releases/download/mobilebert_w8a8_and_w4a8/htp_per_channel_quantsim_config.json" } } diff --git a/aimet_zoo_torch/mobilebert/model/model_definition.py b/aimet_zoo_torch/mobilebert/model/model_definition.py index 1c7cd4d..37001b9 100644 --- a/aimet_zoo_torch/mobilebert/model/model_definition.py +++ b/aimet_zoo_torch/mobilebert/model/model_definition.py @@ -45,7 +45,7 @@ def __init__(self, model_config=None, args=None): AuxArguments, ) self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) - self.cfg = defaultdict(lambda: None) + self.cfg = defaultdict(lambda: None) if model_config: config_filepath = os.path.join( self.parent_dir, "model_cards", model_config + ".json" diff --git a/aimet_zoo_torch/mobilenetv2/model/model_definition.py b/aimet_zoo_torch/mobilenetv2/model/model_definition.py index 8bd458a..26129d9 100644 --- a/aimet_zoo_torch/mobilenetv2/model/model_definition.py +++ b/aimet_zoo_torch/mobilenetv2/model/model_definition.py @@ -9,7 +9,7 @@ # ============================================================================= """Class for downloading and setting up of optimized and original mobilenetv2 model for AIMET model zoo""" -# pylint:disable = import-error, wrong-import-order +# pylint:disable = import-error, wrong-import-order, import-outside-toplevel # adding this due to docker image not setup yet import os @@ -75,7 +75,7 @@ def from_pretrained(self, quantized=False): self.model.load_state_dict(quantized_state_dict["state_dict"]) del quantized_state_dict else: - #pylint:disable=import-outside-toplevel + #pylint:disable = import-outside-toplevel try: from torch.hub import load_state_dict_from_url except ImportError: diff --git a/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py b/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py index e33536f..2e5a5ca 100644 --- a/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py +++ b/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py @@ -63,7 +63,7 @@ def parse_args(raw_args): type=int, default=2022, help="training seed", - ) + ) args = parser.parse_args(raw_args) return args @@ -166,7 +166,7 @@ def main(raw_args=None): logger.info( f"Optimized Model | 8-bit Environment | perplexity: {quantized_model_performance_int8:.4f}" ) - + return { 'original_model_performance_fp32':original_model_performance_fp32, 'original_model_performance_int8':original_model_performance_int8, diff --git a/aimet_zoo_torch/mobilevit/model/model_definition.py b/aimet_zoo_torch/mobilevit/model/model_definition.py index b8c4dc5..0c963b7 100644 --- a/aimet_zoo_torch/mobilevit/model/model_definition.py +++ b/aimet_zoo_torch/mobilevit/model/model_definition.py @@ -55,13 +55,12 @@ def __init__(self, model_config=None, quantized=False): if self.quantized: self.model_name_or_path = os.path.join( self.parent_dir, self.cfg["model_args"]["quantized"]["model_name_or_path"] - ) + ) else: self.model_name_or_path = self.cfg["model_args"]["original"]["model_name_or_path"] - self.config_file = os.path.join( self.parent_dir, self.cfg["model_args"]["config_file"] - ) + ) def get_model_from_pretrained(self): """get original or optmized model diff --git a/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py b/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py index cde8a49..d088eb6 100644 --- a/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py +++ b/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py @@ -49,7 +49,7 @@ def main(raw_args=None): quant_acc = eval_func(model = sim.model.cuda(), dataloader = eval_dataloader) print(f'Quantized quantized accuracy: {quant_acc:0.3f}%') - return {'fp32_acc':fp32_acc, 'quant_acc':quant_acc} + return {'fp32_acc':fp32_acc, 'quant_acc':quant_acc} if __name__ == '__main__': main() diff --git a/aimet_zoo_torch/roberta/model/model_definition.py b/aimet_zoo_torch/roberta/model/model_definition.py index da18569..d0f8231 100644 --- a/aimet_zoo_torch/roberta/model/model_definition.py +++ b/aimet_zoo_torch/roberta/model/model_definition.py @@ -79,7 +79,7 @@ def __init__(self, model_config=None, args=None): self.training_args = training_args self.aux_args = aux_args self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) - self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "roberta_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] diff --git a/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py b/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py index 28afaff..45e050e 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py @@ -322,7 +322,7 @@ def __init__(self, args): DEFAULT_CONFIG = {"num_samples_cal": 500, "num_samples_eval": None} -#pylint:disable = too-many-local-variables +#pylint:disable = too-many-locals def main(raw_args=None): """main evaluation function""" args = arguments(raw_args) @@ -331,6 +331,7 @@ def main(raw_args=None): eval_path = pathlib.Path("./eval_results") eval_path.mkdir(exist_ok=True) + #pylint:disable = consider-using-with class_names = [name.strip() for name in open("voc-model-labels.txt").readlines()] device = get_device(args) #pylint: disable = no-member diff --git a/aimet_zoo_torch/uniformer_classification/evaluators/uniformer_classification_quanteval.py b/aimet_zoo_torch/uniformer_classification/evaluators/uniformer_classification_quanteval.py index 9cb1438..1ae94c1 100644 --- a/aimet_zoo_torch/uniformer_classification/evaluators/uniformer_classification_quanteval.py +++ b/aimet_zoo_torch/uniformer_classification/evaluators/uniformer_classification_quanteval.py @@ -81,9 +81,7 @@ def main(raw_args=None): print(f"Original Model | FP32 Environment | Accuracy: {acc_fp32:.4f}") print(f"Original Model | W{param_bw}A{output_bw} Environment | Accuracy: {acc_orig:.4f}") print(f"Optimized Model | W{param_bw}A{output_bw} Environment | Accuracy: {acc_optim:.4f}") - return {"acc_fp32": acc_fp32, "acc_orig": acc_orig, "acc_optim": acc_optim} if __name__ == "__main__": scores_dict = main() - diff --git a/aimet_zoo_torch/vit/evaluators/vit_quanteval.py b/aimet_zoo_torch/vit/evaluators/vit_quanteval.py index d2e1cf9..21850cf 100644 --- a/aimet_zoo_torch/vit/evaluators/vit_quanteval.py +++ b/aimet_zoo_torch/vit/evaluators/vit_quanteval.py @@ -99,7 +99,7 @@ def main(raw_args=None): accelerator.wait_for_everyone() # get dataset for gathering information to load model - dataset = get_dataset(args) + dataset = get_dataset(args) # loading finetuned original model model = vit(model_config=args.model_config, quantized=False) model_orig = model.get_model_from_pretrained(dataset) diff --git a/aimet_zoo_torch/vit/model/model_definition.py b/aimet_zoo_torch/vit/model/model_definition.py index 1986727..a1a12e9 100644 --- a/aimet_zoo_torch/vit/model/model_definition.py +++ b/aimet_zoo_torch/vit/model/model_definition.py @@ -56,16 +56,15 @@ def __init__(self, model_config=None, quantized=False): if self.quantized: self.model_name_or_path = os.path.join( self.parent_dir, self.cfg["model_args"]["quantized"]["model_name_or_path"] - ) + ) else: self.model_name_or_path = self.cfg["model_args"]["original"]["model_name_or_path"] - self.config_file = os.path.join( self.parent_dir, self.cfg["model_args"]["config_file"] ) self.dataset_name = os.path.join( self.parent_dir, self.cfg["model_args"]["dataset_name"] - ) + ) def get_model_from_pretrained(self,dataset): """get original or optmized model @@ -104,7 +103,6 @@ def get_model_from_pretrained(self,dataset): raise NotImplementedError( "Need to quantize interpolate fn in modeling_vit.py" ) - self.model = VitModel.from_pretrained( self.model_name_or_path, from_tf=bool(".ckpt" in self.model_name_or_path), diff --git a/packaging/pylint_model_zoo.cmake b/packaging/pylint_model_zoo.cmake index 59630ee..c15e8d8 100644 --- a/packaging/pylint_model_zoo.cmake +++ b/packaging/pylint_model_zoo.cmake @@ -10,10 +10,10 @@ message("Preparing Directories for Pylint Results" ...) set(src_results_dir "${SOURCE_DIR}/packaging/results") file(MAKE_DIRECTORY "${SOURCE_DIR}/packaging/results") - message(STATUS "Source Directory: ${SOURCE_DIR}") message(STATUS "Result Directory: ${src_results_dir}") +set(XDG_CACHE_HOME ${SOURCE_DIR}) if(ENABLE_TENSORFLOW) # Add AIMET Tensorflow package to package array list @@ -33,10 +33,11 @@ set(pylint_failed 0) foreach(zoo_pylint_folder IN LISTS pylint_list) message(STATUS "Aimet Model Zoo Variant: ${zoo_pylint_folder}") message(STATUS "Aimet Model Zoo Path Verification: ${SOURCE_DIR}/${zoo_pylint_folder}") + message(STATUS "Sending message format: ${MSG_FORMAT}") execute_process ( - COMMAND pylint --rcfile=${SOURCE_DIR}/.pylintrc -r n ${MSG_FORMAT} ${SOURCE_DIR}/${zoo_pylint_folder} - OUTPUT_VARIABLE pylint_complete + COMMAND bash -c "pylint --rcfile=${SOURCE_DIR}/.pylintrc -r n ${MSG_FORMAT} ${SOURCE_DIR}/${zoo_pylint_folder}/ >> ${src_results_dir}/pylint_${zoo_pylint_folder} && \ + echo 'Pylinting...' && cat ${src_results_dir}/pylint_${zoo_pylint_folder}" RESULT_VARIABLE pylint_return ) message(STATUS "Return Code is: ${pylint_return}") @@ -61,7 +62,4 @@ foreach(zoo_pylint_folder IN LISTS pylint_list) set(pylint_failed 1) endif() - file(WRITE "${src_results_dir}/pylint_${zoo_pylint_folder}" ${pylint_complete}) - message(${pylint_complete}) - endforeach()