Merge back OTE v0.2.0 release fixes

CHANGELOG.md

@@ -2,10 +2,29 @@
 
 All notable changes to this project will be documented in this file.
 
+## \[v0.2.0\]
+### Added
+* Model Preparation Algorithm (MPA), a newly introduced OTE Algorithm backend for advanced transfer learning
+  * Class-Incremental Learning support for OTE models
+    * Image Classification
+    * Object Detection
+    * Semantic Segmentation
+* Object counting & Rotated object detection are added to Object Detection backend
+* Increased support for NNCF / FP16 / HPO
+* Ignored label support
+* Stop training on NaN losses
+
+
+### Changed
+* Major refactoring
+  * Tasks & model templates had been moved to OTE repo from each OTE Algorithm backend
+
+
 ## \[v0.1.1\]
 ### Fixed
 * Some minor issues
 
+
 ## \[v0.1.0\]
 ### Added
 * OTE SDK, defines an interface which can be used by OTE CLI to access OTE Algorithms.
@@ -16,10 +35,3 @@ All notable changes to this project will be documented in this file.
   * Object Detection
   * Semantic Segmentation
 
-## \[v0.2.0\]
-### Added
-* Model Preparation Algorithm (MPA), a newly introduced OTE Algorithm backend for advanced transfer learning
-  * Class-Incremental Learning support for OTE models
-    * Image Classification
-    * Object Detection
-    * Semantic Segmentation

external/mmdetection/configs/custom-counting-instance-seg/efficientnetb2b_maskrcnn/model.py

@@ -157,3 +157,4 @@
     dict(type='EarlyStoppingHook', patience=10, metric='mAP',
          interval=1, priority=75, iteration_patience=0),
 ]
+fp16 = dict(loss_scale=512.)

external/mmdetection/tests/expected_metrics/metrics_test_ote_training.yml

@@ -349,3 +349,47 @@
 : "metrics.accuracy.f-measure":
     "base": "nncf_evaluation.metrics.accuracy.f-measure"
     "max_diff": 0.06
+
+? "ACTION-training_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_ResNet50,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "target_value": 0.92
+    "max_diff_if_less_threshold": 0.06
+    "max_diff_if_greater_threshold": 0.06
+? "ACTION-export_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_ResNet50,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01
+? "ACTION-pot_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_ResNet50,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01
+? "ACTION-nncf_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_ResNet50,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff_if_less_threshold": 0.01
+? "ACTION-nncf_export_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_ResNet50,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "nncf_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01
+
+? "ACTION-training_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_EfficientNetB2B,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "target_value": 0.92
+    "max_diff_if_less_threshold": 0.06
+    "max_diff_if_greater_threshold": 0.06
+? "ACTION-export_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_EfficientNetB2B,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01
+? "ACTION-pot_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_EfficientNetB2B,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01
+? "ACTION-nncf_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_EfficientNetB2B,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff_if_less_threshold": 0.01
+? "ACTION-nncf_export_evaluation,model-Custom_Counting_Instance_Segmentation_MaskRCNN_EfficientNetB2B,dataset-aeromonas,num_iters-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "nncf_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01

external/mmdetection/tests/ote_cli/test_instance_segmentation.py

@@ -34,6 +34,13 @@
     ote_eval_testing,
     ote_train_testing,
     ote_export_testing,
+    nncf_optimize_testing,
+    nncf_export_testing,
+    nncf_eval_testing,
+    ote_hpo_testing,
+    nncf_eval_openvino_testing,
+    pot_optimize_testing,
+    pot_eval_testing
 )
 
 
@@ -111,3 +118,45 @@ def test_ote_eval_deployment(self, template):
     @pytest.mark.parametrize("template", templates, ids=templates_ids)
     def test_ote_demo_deployment(self, template):
         ote_demo_deployment_testing(template, root, ote_dir, args)
+
+    @e2e_pytest_component
+    @pytest.mark.parametrize("template", templates, ids=templates_ids)
+    def test_nncf_optimize(self, template):
+        if template.entrypoints.nncf is None:
+            pytest.skip("nncf entrypoint is none")
+
+        nncf_optimize_testing(template, root, ote_dir, args)
+
+    @e2e_pytest_component
+    @pytest.mark.parametrize("template", templates, ids=templates_ids)
+    def test_nncf_export(self, template):
+        if template.entrypoints.nncf is None:
+            pytest.skip("nncf entrypoint is none")
+
+        nncf_export_testing(template, root)
+
+    @e2e_pytest_component
+    @pytest.mark.parametrize("template", templates, ids=templates_ids)
+    def test_nncf_eval(self, template):
+        if template.entrypoints.nncf is None:
+            pytest.skip("nncf entrypoint is none")
+
+        nncf_eval_testing(template, root, ote_dir, args, threshold=0.001)
+
+    @e2e_pytest_component
+    @pytest.mark.parametrize("template", templates, ids=templates_ids)
+    def test_nncf_eval_openvino(self, template):
+        if template.entrypoints.nncf is None:
+            pytest.skip("nncf entrypoint is none")
+
+        nncf_eval_openvino_testing(template, root, ote_dir, args)
+
+    @e2e_pytest_component
+    @pytest.mark.parametrize("template", templates, ids=templates_ids)
+    def test_pot_optimize(self, template):
+        pot_optimize_testing(template, root, ote_dir, args)
+
+    @e2e_pytest_component
+    @pytest.mark.parametrize("template", templates, ids=templates_ids)
+    def test_pot_eval(self, template):
+        pot_eval_testing(template, root, ote_dir, args)