Merge pull request #898 from openvinotoolkit/pf/adding_anomaly_training_tests

[Anomaly] Added real-life training tests

Author: sstrehlk

external/anomaly/ote_anomalib/nncf_task.py

@@ -36,6 +36,7 @@
 from ote_sdk.entities.datasets import DatasetEntity
 from ote_sdk.entities.model import (
     ModelEntity,
+    ModelFormat,
     ModelOptimizationType,
     ModelPrecision,
     OptimizationMethod,
@@ -183,6 +184,8 @@ def optimize(
         self.trainer = Trainer(**self.config.trainer, logger=False, callbacks=callbacks)
         self.trainer.fit(model=self.model, datamodule=datamodule)
         self.compression_ctrl = nncf_callback.nncf_ctrl
+        output_model.model_format = ModelFormat.BASE_FRAMEWORK
+        output_model.optimization_type = ModelOptimizationType.NNCF
         self.save_model(output_model)
 
         logger.info("Training completed.")

external/anomaly/ote_tests_pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+python_files = test_ote_api.py test_ote_inference.py test_ote_training.py

external/anomaly/tests/conftest.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+import pytest
+from ote_sdk.test_suite.pytest_insertions import (
+    get_pytest_plugins_from_ote,
+    ote_conftest_insertion,
+    ote_pytest_generate_tests_insertion,
+    ote_pytest_addoption_insertion,
+)
+from ote_sdk.test_suite.training_tests_common import REALLIFE_USECASE_CONSTANT
+
+pytest_plugins = get_pytest_plugins_from_ote()
+
+ote_conftest_insertion(default_repository_name="ote/training_extensions/external/anomaly")
+
+
+@pytest.fixture
+def ote_test_domain_fx():
+    return "custom-anomaly-classification"
+
+
+@pytest.fixture
+def ote_test_scenario_fx(current_test_parameters_fx):
+    assert isinstance(current_test_parameters_fx, dict)
+    if current_test_parameters_fx.get("usecase") == REALLIFE_USECASE_CONSTANT:
+        return "performance"
+    else:
+        return "integration"
+
+
+@pytest.fixture(scope="session")
+def ote_templates_root_dir_fx():
+    import os.path as osp
+    import logging
+
+    logger = logging.getLogger(__name__)
+    root = osp.dirname(osp.dirname(osp.realpath(__file__)))
+    root = f"{root}/anomaly_classification/"
+    logger.debug(f"overloaded ote_templates_root_dir_fx: return {root}")
+    return root
+
+
+@pytest.fixture(scope="session")
+def ote_reference_root_dir_fx():
+    import os.path as osp
+    import logging
+
+    logger = logging.getLogger(__name__)
+    root = osp.dirname(osp.dirname(osp.realpath(__file__)))
+    root = f"{root}/tests/reference/"
+    logger.debug(f"overloaded ote_reference_root_dir_fx: return {root}")
+    return root
+
+
+# pytest magic
+def pytest_generate_tests(metafunc):
+    ote_pytest_generate_tests_insertion(metafunc)
+
+
+def pytest_addoption(parser):
+    ote_pytest_addoption_insertion(parser)

external/anomaly/tests/expected_metrics/metrics_test_ote_training.yml

@@ -0,0 +1,43 @@
+? "ACTION-training_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "target_value": 0.73
+    "max_diff_if_less_threshold": 0.01
+    "max_diff_if_greater_threshold": 0.04
+? "ACTION-export_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01
+? "ACTION-pot_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "export_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.03
+? "ACTION-nncf_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff_if_less_threshold": 0.01
+? "ACTION-nncf_export_evaluation,model-ote_anomaly_classification_padim,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "nncf_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01
+
+? "ACTION-training_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "target_value": 0.74
+    "max_diff_if_less_threshold": 0.01
+    "max_diff_if_greater_threshold": 0.04
+? "ACTION-export_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01
+? "ACTION-pot_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "export_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.03
+? "ACTION-nncf_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "training_evaluation.metrics.accuracy.f-measure"
+    "max_diff_if_less_threshold": 0.01
+? "ACTION-nncf_export_evaluation,model-ote_anomaly_classification_stfpm,dataset-mvtec,patience-CONFIG,batch-CONFIG,usecase-reallife"
+: "metrics.accuracy.f-measure":
+    "base": "nncf_evaluation.metrics.accuracy.f-measure"
+    "max_diff": 0.01