Added more acceptance test coverage and minor fixes to existing tests…

…; Updates to CI/CD pipeline; Added 3 new TF models and updated Torch MobileBert

- Minor clean-up of code to minimize errors from static analysis (pylint)
- Added more acceptance tests for a few pytorch segmentation models and applied minor fixes to previous acceptance tests
- Config file and documentation updates for a pytorch MobileBert models
- Releasing next package release of aimet model zoo (installable wheel file binaries)
- Added code, documentation and artifacts the following new Tensorflow models: MobileNet-EdgeTPU, DeepLabV3Plus_Xception_TF2 and DeepLabV3Plus_MBNV2_TF2

Signed-off-by: Hanwen Xiong <quic_hanwxion@quicinc.com>

.pylintrc

@@ -558,4 +558,4 @@ valid-metaclass-classmethod-first-arg=mcs
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=base.Exception

AcceptanceTests/CMakeLists.txt

@@ -15,16 +15,18 @@ if (ENABLE_TORCH)
 
 endif (ENABLE_TORCH)
 
-#if (ENABLE_TENSORFLOW)
-#    add_dependencies(AcceptanceTests
-#            AcceptanceTests.Tensorflow)
-#
-#endif (ENABLE_TENSORFLOW)
+if (ENABLE_TENSORFLOW)
+    add_dependencies(AcceptanceTests
+            AcceptanceTests.Tensorflow)
+
+endif (ENABLE_TENSORFLOW)
 
 if (ENABLE_TORCH)
+    message(STATUS "Torch has been enabled")
     add_subdirectory(torch)
 endif (ENABLE_TORCH)
 
-#if (ENABLE_TENSORFLOW)
-#    add_subdirectory(tensorflow)
-#endif (ENABLE_TENSORFLOW)
+if (ENABLE_TENSORFLOW)
+    message(STATUS "Tensorflow has been enabled")
+    add_subdirectory(tensorflow)
+endif (ENABLE_TENSORFLOW)

AcceptanceTests/tensorflow/CMakeLists.txt

@@ -0,0 +1,32 @@
+# =============================================================================
+#  @@-COPYRIGHT-START-@@
+#
+#  Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+#
+#  @@-COPYRIGHT-END-@@
+# =============================================================================
+if (ENABLE_CUDA)
+    set(CUDA_FLAG -m "not blah")
+    set(USE_CUDA True)
+else (ENABLE_CUDA)
+    set(CUDA_FLAG -m "not cuda")
+    set(USE_CUDA False)
+endif (ENABLE_CUDA)
+
+add_custom_target( AcceptanceTests.Tensorflow )
+
+file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/*quanteval.py")
+    foreach(filename ${files})
+        get_filename_component( testname "${filename}" NAME_WE )
+        string(SUBSTRING ${testname} 5 -1 formatted_name)
+        message(STATUS  "Now running Tensorflow Test: ${formatted_name}")
+
+        add_custom_target(AcceptanceTests.Tensorflow.${testname}
+            VERBATIM COMMAND . /opt/${formatted_name}-venv/bin/activate && export HF_HOME=/opt/cache &&
+                             ${CMAKE_COMMAND} -E env
+                             ${MZ_PYTHONPATH}:$$PYTHONPATH 
+                             python -m pytest -s ${filename} 
+                             ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml) 
+
+        add_dependencies( AcceptanceTests.Tensorflow AcceptanceTests.Tensorflow.${testname} )
+    endforeach( filename )

AcceptanceTests/tensorflow/conftest.py

@@ -52,4 +52,20 @@ def tiny_mscoco_tfrecords(test_data_path):
 
     yield tiny_mscoco_tfrecords
 
+@pytest.fixture(scope='module')
+def PascalVOC_segmentation_test_data_path(test_data_path):
+    if test_data_path is not None:
+        pascalVOC_segmentation_path = (test_data_path / "model_zoo_datasets/tiny_VOCdevkit").as_posix() 
+    else:
+        pascalVOC_segmentation_path = None
+
+    yield pascalVOC_segmentation_path
+
+#@pytest.fixture(scope='module')
+#def tiny_imageNet_tfrecords(test_data_path):
+#    if test_data_path is not None:
+#        tiny_imageNet_tfrecords = (test_data_path / "model_zoo_datasets/tiny_imageNet_tfrecords").as_posix() 
+#    else:
+#        tiny_imageNet_tfrecords = None
 
+#    yield tiny_imageNet_tfrecords

AcceptanceTests/tensorflow/staging/test_deeplabv3plus_mbnv2_tf2_quanteval.py

@@ -0,0 +1,37 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+
+""" acceptance test for deeplabv3plus mbnv2"""
+
+import pytest
+from aimet_zoo_tensorflow.deeplabv3plus_tf2.evaluators import deeplabv3plus_tf2_quanteval
+
+@pytest.mark.slow 
+@pytest.mark.cuda 
+@pytest.mark.sementic_segmentation
+# pylint:disable = redefined-outer-name
+@pytest.mark.parametrize("model_config", ["deeplabv3plus_mbnv2_w8a8"])
+def test_quanteval_deeplabv3plus_mbnv2_tf2(model_config, PascalVOC_segmentation_test_data_path):
+    """mobiledet edgetpu image classification test"""
+
+    if PascalVOC_segmentation_test_data_path is None:
+        pytest.xfail(f'Dataset path is not set')
+
+    deeplabv3plus_tf2_quanteval.main(
+        [
+            "--model-config",
+            model_config,
+            "--dataset-path",
+            PascalVOC_segmentation_test_data_path
+        ]
+    )
+

AcceptanceTests/tensorflow/staging/test_deeplabv3plus_xception_tf2_quanteval.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+#  @@-COPYRIGHT-START-@@
+#
+#  Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+#
+#  @@-COPYRIGHT-END-@@
+# =============================================================================
+
+""" acceptance test for deeplabv3plus xception"""
+
+import pytest
+from aimet_zoo_tensorflow.deeplabv3plus_tf2.evaluators import deeplabv3plus_tf2_quanteval
+
+@pytest.mark.slow 
+@pytest.mark.cuda 
+@pytest.mark.sementic_segmentation
+# pylint:disable = redefined-outer-name
+@pytest.mark.parametrize("model_config", ["deeplabv3plus_xception_w8a8"])
+def test_quanteval_deeplabv3plus_xception_tf2(model_config, PascalVOC_segmentation_test_data_path):
+    """mobiledet edgetpu image classification test"""
+
+    if PascalVOC_segmentation_test_data_path is None:
+        pytest.xfail(f'Dataset path is not set')
+
+    deeplabv3plus_tf2_quanteval.main(
+        [
+            "--model-config",
+            model_config,
+            "--dataset-path",
+            PascalVOC_segmentation_test_data_path
+        ]
+    )
+

AcceptanceTests/tensorflow/staging/test_mobilenet_edgetpu_quanteval.py

@@ -0,0 +1,34 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+
+""" acceptance test for resnet"""
+import pytest
+from aimet_zoo_tensorflow.mobilenetedgetpu.evaluators import mobilenet_edgetpu_quanteval
+
+@pytest.mark.cuda
+@pytest.mark.image_classification
+# pylint:disable = redefined-outer-name
+@pytest.mark.parametrize("model_config", ["mobilenetedgetpu_w8a8"])
+def test_quanteval_mobilenet_edgetpu(model_config, tiny_imageNet_tfrecords):
+    """resnet image classification test"""
+
+    if tiny_imageNet_tfrecords is None:
+        pytest.xfail(f'failed since dataset path is not set')
+
+    mobilenet_edgetpu_quanteval.main(
+        [
+            "--model-config",
+            model_config,
+            "--dataset-path",
+            tiny_imageNet_tfrecords,
+        ]
+    )

AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py

@@ -18,7 +18,7 @@
 @pytest.mark.cuda 
 @pytest.mark.object_detection
 # pylint:disable = redefined-outer-name
-@pytest.mark.parametrize("model_config", ["resnet50_w8a8"])
+@pytest.mark.parametrize("model_config", ["mobilenetv2_w8a8"])
 def test_quanteval_mobilenet_v2(model_config, tiny_imageNet_root_path):
     """mobilenet_v2_tf2 image classification acceptance test"""
 

AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py

@@ -13,7 +13,7 @@
 """ acceptance test for ssd_mobilenetv2_quanteval edgetpu"""
 
 import pytest
-from aimet_zoo_tensorflow.ssd_mobilenet_v2.evaluators import ssd_mobilenetv2_quanteval
+from aimet_zoo_tensorflow.ssd_mobilenet_v2.evaluators import ssd_mobilenet_v2_quanteval
 
 @pytest.mark.slow 
 @pytest.mark.cuda 
@@ -26,7 +26,7 @@ def test_quanteval_ssd_mobilenetv2(model_config, tiny_mscoco_tfrecords):
     if tiny_mscoco_tfrecords is None:
         pytest.fail(f'Dataset path is not set')
 
-    ssd_mobilenetv2_quanteval.main(
+    ssd_mobilenet_v2_quanteval.main(
         [
             "--model-config",
             model_config,

AcceptanceTests/torch/CMakeLists.txt

@@ -15,16 +15,17 @@ endif (ENABLE_CUDA)
 
 add_custom_target( AcceptanceTests.Torch )
 
-file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/test_resnet_quanteval.py")
+file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/*quanteval.py")
     foreach(filename ${files})
-
         get_filename_component( testname "${filename}" NAME_WE )
-        message(STATUS "Testname: " testname)
-
+        string(SUBSTRING ${testname} 5 -1 formatted_name)
+        message(STATUS  "Now running Pytorch Test: ${formatted_name}")
+
         add_custom_target(AcceptanceTests.Torch.${testname}
-            VERBATIM COMMAND ${CMAKE_COMMAND} -E env
-                             ${MZ_PYTHONPATH} 
-                             ${Python3_EXECUTABLE} -m pytest -s ${filename} 
+            VERBATIM COMMAND . /opt/${formatted_name}-venv/bin/activate && export HF_HOME=/opt/cache &&
+                             ${CMAKE_COMMAND} -E env
+                             ${MZ_PYTHONPATH}:$$PYTHONPATH 
+                             python -m pytest -s ${filename} 
                              ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml) 
 
         add_dependencies( AcceptanceTests.Torch AcceptanceTests.Torch.${testname} )

AcceptanceTests/torch/conftest.py

@@ -69,7 +69,7 @@ def tiny_mscoco_validation_path(test_data_path):
     else:
         tiny_mscoco_validation_path = None
 
-    yield tiny_mscoco_validation_path.as_posix()
+    yield tiny_mscoco_validation_path
 
 
 @pytest.fixture(scope='module')

AcceptanceTests/torch/pytest.ini

@@ -15,5 +15,5 @@ markers =
     nlp: tests that belong to natual language process task
     object_detection: tests that belong to object detection task
     pose_estimation: tests that belong to pose estimation task
-    sementic_segmentation: tests that belong to sementic segmentation task
+    semantic_segmentation: tests that belong to sementic segmentation task
     super_resolution: tests that belong to super resolution task  

AcceptanceTests/torch/staging/test_ffnet_quanteval.py

@@ -0,0 +1,65 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+""" acceptance test for ffnet semantic segmentation"""
+import pytest
+from math import isclose
+
+import torch
+
+from aimet_zoo_torch.ffnet.evaluators import (
+    ffnet_quanteval,
+)
+
+expected_results = {
+    'segmentation_ffnet40S_dBBB_mobile': {'original_mIoU': 0.7015, 'quantized_mIoU': 0.7018},
+    'segmentation_ffnet54S_dBBB_mobile': {'original_mIoU': 0.6957, 'quantized_mIoU': 0.7368},
+    'segmentation_ffnet78S_BCC_mobile_pre_down': {'original_mIoU': None, 'quantized_mIoU': None},
+    'segmentation_ffnet78S_dBBB_mobile': {'original_mIoU': 0.6904, 'quantized_mIoU': 0.6882},
+    'segmentation_ffnet122NS_CCC_mobile_pre_down': {'original_mIoU': None, 'quantized_mIoU': None}
+}
+
+@pytest.mark.sementic_segmentation 
+@pytest.mark.cuda
+#pylint:disable = redefined-outer-name
+@pytest.mark.parametrize(
+    "model_config, expected_mIoUs",[
+       ("segmentation_ffnet40S_dBBB_mobile", expected_results["segmentation_ffnet40S_dBBB_mobile"]),
+       ("segmentation_ffnet54S_dBBB_mobile", expected_results["segmentation_ffnet54S_dBBB_mobile"]),
+       ("segmentation_ffnet78S_BCC_mobile_pre_down", expected_results["segmentation_ffnet78S_BCC_mobile_pre_down"]),
+       ("segmentation_ffnet78S_dBBB_mobile", expected_results["segmentation_ffnet78S_dBBB_mobile"]),
+       ("segmentation_ffnet122NS_CCC_mobile_pre_down", expected_results["segmentation_ffnet122NS_CCC_mobile_pre_down"])
+       ]
+    )
+def test_quaneval_ffnet(
+        model_config,
+        expected_mIoUs,
+        tiny_cityscapes_path
+):
+    """acceptance test of hrnet for semantic segmentation"""
+    torch.cuda.empty_cache()
+    if tiny_cityscapes_path is None:
+        pytest.xfail('Dataset is not set')
+
+    #TODO: Fix the two failing model cards
+    if expected_mIoUs['original_mIoU'] is None:
+        pytest.skip(f'{model_config} hasn`t passed manual testing!')
+
+    mIoUs = ffnet_quanteval.main(
+        [
+            "--model-config", model_config,
+            "--dataset-path", tiny_cityscapes_path,
+            "--batch-size", '2'
+        ]
+    )
+
+    assert isclose(mIoUs['mIoU_orig_fp32'], expected_mIoUs['original_mIoU'], rel_tol=1e-4)
+    assert isclose(mIoUs['mIoU_optim_int8'], expected_mIoUs['quantized_mIoU'], rel_tol=1e-4)

AcceptanceTests/torch/test_hrnet_posenet_quanteval.py → AcceptanceTests/torch/staging/test_hrnet_posenet_quanteval.py

@@ -18,7 +18,7 @@
 
 @pytest.mark.pose_estimation
 @pytest.mark.cuda
-@pytest.parametrize("model_config",["hrnet_posenet_w4a8","hrnet_posenet_w8a8"])
+@pytest.mark.parametrize("model_config",["hrnet_posenet_w4a8","hrnet_posenet_w8a8"])
 def test_quaneval_hrnet_posenet(model_config, tiny_mscoco_validation_path):
     """hrnet_posenet pose estimation test"""
     torch.cuda.empty_cache()

AcceptanceTests/torch/test_hrnet_sem_seg_quanteval.py → AcceptanceTests/torch/staging/test_hrnet_sem_seg_quanteval.py

@@ -23,7 +23,7 @@
 @pytest.mark.sementic_segmentation 
 @pytest.mark.cuda
 #pylint:disable = redefined-outer-name
-@pytest.mark.parametrize("model_config",["hrnet_sem_seg_w4a8","hrnet_sem_seg_w4a8"])
+@pytest.mark.parametrize("model_config",["hrnet_sem_seg_w4a8"])
 def test_quaneval_hrnet_sem_seg(model_config, tiny_cityscapes_path, monkeypatch):
     """acceptance test of hrnet for semantic segmentation"""
     torch.cuda.empty_cache()

AcceptanceTests/torch/staging/test_inverseform_quanteval.py

@@ -0,0 +1,41 @@
+# /usr/bin/env python3
+# -*- mode: python -*-
+# =============================================================================
+# @@-COPYRIGHT-START-@@
+#
+# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
+# Changes from QuIC are licensed under the terms and conditions at
+# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
+#
+# @@-COPYRIGHT-END-@@
+# =============================================================================
+""" acceptance test for inverseform """
+import pytest
+from math import isclose
+
+from aimet_zoo_torch.inverseform.evaluators.inverseform_quanteval import main
+
+expected_results = {
+    'hrnet_16_slim_if': {'original_mIoU': 0.6883, 'quantized_mIoU': 0.6850},
+    'ocrnet_48_if': {'original_mIoU': 0.8499, 'quantized_mIoU': 0.8465}
+}
+
+
+@pytest.mark.semantic_segmentation
+@pytest.mark.parametrize("model_config, expected_mIoUs", [('hrnet_16_slim_if', expected_results['hrnet_16_slim_if']),
+                                                          ('ocrnet_48_if', expected_results['ocrnet_48_if'])])
+def test_inverseform_quanteval(
+        model_config,
+        expected_mIoUs,
+        tiny_cityscapes_path
+):
+    if tiny_cityscapes_path is None:
+        pytest.xfail(f"dataset path is None!")
+
+    args = ['--model-config', model_config,
+            '--dataset-path', tiny_cityscapes_path,
+            '--batch-size', '2']
+    mIoUs = main(args)
+
+    assert isclose(mIoUs['original_mIoU'], expected_mIoUs['original_mIoU'] ,rel_tol=1e-4)
+    assert isclose(mIoUs['quantized_mIoU'], expected_mIoUs['quantized_mIoU'], rel_tol=1e-4)