Merge back OTX2.2RC12 to OTX2.3

.ci/docker/Dockerfile

@@ -5,16 +5,9 @@
 FROM nvidia/cuda:12.1.0-runtime-ubuntu20.04@sha256:c1869c30f46fff478a37ed58d9dace7e08519541274f03424d0b78bd35b2c73a AS python_base_cuda
 LABEL maintainer="OpenVINO Training Extensions Development Team"
 
-ARG HTTP_PROXY
-ARG HTTPS_PROXY
-ARG NO_PROXY
 ARG uid
 ARG gid
 
-# Setup proxies
-ENV http_proxy=$HTTP_PROXY
-ENV https_proxy=$HTTPS_PROXY
-ENV no_proxy=$NO_PROXY
 ENV DEBIAN_FRONTEND="noninteractive"
 
 # hadolint ignore=DL3008

.ci/docker/build.sh

@@ -53,9 +53,6 @@ fi
 TAG=$1
 
 docker build -f ./Dockerfile \
---build-arg HTTP_PROXY="${http_proxy:?}" \
---build-arg HTTPS_PROXY="${https_proxy:?}" \
---build-arg NO_PROXY="${no_proxy:?}" \
 --build-arg ACTIONS_RUNNER_VER="$ACTIONS_RUNNER_VER" \
 --build-arg gid="$(id -g)" \
 --build-arg uid="$UID" \

.ci/docker/start-runner.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 GPU_ID="all"
-VER_CUDA="11.7.1"
+VER_CUDA="12.1.0"
 TAG_RUNNER="latest"
 ADDITIONAL_LABELS=""
 MOUNT_PATH=""
@@ -149,9 +149,6 @@ if [ "$DEBUG_CONTAINER" = true ]; then
         --name "$CONTAINER_NAME" \
         -e NVIDIA_VISIBLE_DEVICES="$GPU_ID" \
         ${ENV_FLAGS} \
-        -e http_proxy=http://proxy-chain.intel.com:911 \
-        -e https_proxy=http://proxy-chain.intel.com:912 \
-        -e no_proxy=intel.com,.intel.com,localhost,127.0.0.0/8 \
         ${MOUNT_FLAGS} \
         ${CACHE_MOUNT_FLAGS} \
         "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:"$TAG_RUNNER"; RET=$?
@@ -172,9 +169,6 @@ else
         --name "$CONTAINER_NAME" \
         -e NVIDIA_VISIBLE_DEVICES="$GPU_ID" \
         ${ENV_FLAGS} \
-        -e http_proxy=http://proxy-chain.intel.com:911 \
-        -e https_proxy=http://proxy-chain.intel.com:912 \
-        -e no_proxy=intel.com,.intel.com,localhost,127.0.0.0/8 \
         ${MOUNT_FLAGS} \
         ${CACHE_MOUNT_FLAGS} \
         "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:"$TAG_RUNNER"; RET=$?

.github/workflows/pre_merge.yaml

@@ -101,9 +101,6 @@ jobs:
           - task: "multi_cls_classification"
           - task: "multi_label_classification"
           - task: "hlabel_classification"
-          - task: "detection"
-          - task: "instance_segmentation"
-          - task: "semantic_segmentation"
           - task: "visual_prompting"
           - task: "zero_shot_visual_prompting"
           - task: "anomaly_classification"
@@ -127,3 +124,32 @@ jobs:
           rm /tmp/requirements.txt
       - name: Run Integration Test
         run: tox -vv -e integration-test-${{ matrix.task }}
+  Integration-Test-Large:
+    if: |
+      github.event.pull_request.draft == false &&
+      !(startsWith(github.event.pull_request.title, '[WIP]'))
+    runs-on: [self-hosted, linux, x64, dev, dmount]
+    needs: Unit-Test
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - task: "detection"
+          - task: "instance_segmentation"
+          - task: "semantic_segmentation"
+    name: Integration-Test-Large-${{ matrix.task }}-py310
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Install Python
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+        with:
+          python-version: "3.10"
+      - name: Install tox
+        run: |
+          python -m pip install --require-hashes --no-deps -r .ci/requirements.txt
+          pip-compile --generate-hashes --output-file=/tmp/requirements.txt --extra=ci_tox pyproject.toml
+          python -m pip install --require-hashes --no-deps -r /tmp/requirements.txt
+          rm /tmp/requirements.txt
+      - name: Run Integration Test
+        run: tox -vv -e integration-test-${{ matrix.task }}

CHANGELOG.md

@@ -40,29 +40,29 @@ All notable changes to this project will be documented in this file.
 ### New features
 
 - Add RT-DETR model for Object Detection
-  (https://github.com/openvinotoolkit/training_extensions/pull/3741)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3741>)
 - Add Multi-Label & H-label Classification with torchvision models
-  (https://github.com/openvinotoolkit/training_extensions/pull/3697)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3697>)
 - Add Hugging-Face Model Wrapper for Classification
-  (https://github.com/openvinotoolkit/training_extensions/pull/3710)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3710>)
 - Add LoRA finetuning capability for ViT Architectures
-  (https://github.com/openvinotoolkit/training_extensions/pull/3729)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3729>)
 - Add Hugging-Face Model Wrapper for Object Detection
-  (https://github.com/openvinotoolkit/training_extensions/pull/3747)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3747>)
 - Add Hugging-Face Model Wrapper for Semantic Segmentation
-  (https://github.com/openvinotoolkit/training_extensions/pull/3749)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3749>)
 - Enable torch.compile to work with classification
-  (https://github.com/openvinotoolkit/training_extensions/pull/3758)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3758>)
 - Add `otx benchmark` subcommand
-  (https://github.com/openvinotoolkit/training_extensions/pull/3762)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3762>)
 - Add RTMPose for Keypoint Detection Task
-  (https://github.com/openvinotoolkit/training_extensions/pull/3781)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3781>, <https://github.com/openvinotoolkit/training_extensions/pull/4034>)
 - Add Semi-SL MeanTeacher algorithm for Semantic Segmentation
-  (https://github.com/openvinotoolkit/training_extensions/pull/3801)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3801>)
 - Update head and h-label format for hierarchical label classification
-  (https://github.com/openvinotoolkit/training_extensions/pull/3810)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3810>)
 - Support configurable input size
-  (https://github.com/openvinotoolkit/training_extensions/pull/3788)
+  (<https://github.com/openvinotoolkit/training_extensions/pull/3788>)
 
 ### Enhancements
 
@@ -90,9 +90,21 @@ All notable changes to this project will be documented in this file.
   (<https://github.com/openvinotoolkit/training_extensions/pull/3968>)
 - Change sematic segmentation to consider bbox only annotations
   (<https://github.com/openvinotoolkit/training_extensions/pull/3996>)
+- Relieve memory usage criteria on batch size 2 during adaptive batch size
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4009>)
+- Remove background label from RT Info for segmentation task
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4011>)
+- Prevent using too low confidence thresholds in detection
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4018>)
+- Update HPO interface
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4035>)
+- Bump onnx to 1.17.0 to omit CVE-2024-5187
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4063>)
 
 ### Bug fixes
 
+- Update anomaly base transforms to use square resizing
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4059>)
 - Fix Combined Dataloader & unlabeled warmup loss in Semi-SL
   (<https://github.com/openvinotoolkit/training_extensions/pull/3723>)
 - Revert #3579 to fix issues with replacing coco_instance with a different format in some dataset
@@ -105,6 +117,30 @@ All notable changes to this project will be documented in this file.
   (<https://github.com/openvinotoolkit/training_extensions/pull/3946>)
 - Fix config converter for tiling
   (<https://github.com/openvinotoolkit/training_extensions/pull/3973>)
+- Fix `BboxOverlaps2D` handling of empty ground-truth annotations in datasets.
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4010>)
+- Fix num_trials calculation on dataset length less than num_class
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4014>)
+- Fix out_features in HierarchicalCBAMClsHead
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4016>)
+- Fix multilabel_accuracy of MixedHLabelAccuracy
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4042>)
+- Fix wrong indices setting in HLabelInfo
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4044>)
+- Add legacy template LiteHRNet_18 template
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4049>)
+- Model templates: rename model_status value 'DISCONTINUED' to 'OBSOLETE'
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4051>)
+- Enable export of feature vectors for semantic segmentation task
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4055>)
+- Update MRCNN model export to include feature vector and saliency map
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4056>)
+- Upgrade MAPI in 2.2
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4052>)
+- Fix applying model's hparams when loading model from checkpoint
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4057>)
+- Fix incorrect all_groups order configuration in HLabelInfo
+  (<https://github.com/openvinotoolkit/training_extensions/pull/4067>)
 
 ## \[v2.1.0\]
 

README.md

@@ -197,6 +197,9 @@ In addition to the examples above, please refer to the documentation for tutoria
 - Include full image with anno in case there's no tile in tile dataset
 - Add type checker in converter for callable functions (optimizer, scheduler)
 - Change sematic segmentation to consider bbox only annotations
+- Relieve memory usage criteria on batch size 2 during adaptive batch size
+- Remove background label from RT Info for segmentation task
+- Prevent using too low confidence thresholds in detection
 
 ### Bug fixes
 
@@ -206,6 +209,10 @@ In addition to the examples above, please refer to the documentation for tutoria
 - Add missing tile recipes and various tile recipe changes
 - Change categories mapping logic
 - Fix config converter for tiling
+- Fix num_trials calculation on dataset length less than num_class
+- Fix out_features in HierarchicalCBAMClsHead
+- Fix multilabel_accuracy of MixedHLabelAccuracy
+- Fix wrong indices setting in HLabelInfo
 
 ### Known issues
 

docs/source/guide/release_notes/index.rst

@@ -4,7 +4,7 @@ Releases
 .. toctree::
   :maxdepth: 1
 
-v2.2.0 (2024.09)
+v2.2.0 (2024.10)
 ----------------
 
 New features
@@ -38,6 +38,9 @@ Enhancements
 - Include full image with anno in case there's no tile in tile dataset
 - Add type checker in converter for callable functions (optimizer, scheduler)
 - Change sematic segmentation to consider bbox only annotations
+- Relieve memory usage criteria on batch size 2 during adaptive batch size
+- Remove background label from RT Info for segmentation task
+- Prevent using too low confidence thresholds in detection
 
 Bug fixes
 ^^^^^^^^^
@@ -48,6 +51,10 @@ Bug fixes
 - Add missing tile recipes and various tile recipe changes
 - Change categories mapping logic
 - Fix config converter for tiling
+- Fix num_trials calculation on dataset length less than num_class
+- Fix out_features in HierarchicalCBAMClsHead
+- Fix multilabel_accuracy of MixedHLabelAccuracy
+- Fix wrong indices setting in HLabelInfo
 
 v2.1.0 (2024.07)
 ----------------

pyproject.toml

@@ -81,8 +81,8 @@ xpu = [
     "timm==1.0.3",
     "openvino==2024.4",
     "openvino-dev==2024.4",
-    "openvino-model-api==0.2.4",
-    "onnx==1.16.2",
+    "openvino-model-api==0.2.5",
+    "onnx==1.17.0",
     "onnxconverter-common==1.14.0",
     "nncf==2.13.0",
     "anomalib[core]==1.1.0",
@@ -96,7 +96,7 @@ base = [
     "openvino==2024.4",
     "openvino-dev==2024.4",
     "openvino-model-api==0.2.4",
-    "onnx==1.16.2",
+    "onnx==1.17.0",
     "onnxconverter-common==1.14.0",
     "nncf==2.13.0",
     "anomalib[core]==1.1.0",

src/otx/algo/classification/heads/hlabel_cls_head.py

@@ -355,7 +355,7 @@ def __init__(
         self.fc_superclass = nn.Linear(in_channels * self.step_size[0] * self.step_size[1], num_multiclass_heads)
         self.attention_fc = nn.Linear(num_multiclass_heads, in_channels * self.step_size[0] * self.step_size[1])
         self.cbam = CBAM(in_channels)
-        self.fc_subclass = nn.Linear(in_channels * self.step_size[0] * self.step_size[1], num_single_label_classes)
+        self.fc_subclass = nn.Linear(in_channels * self.step_size[0] * self.step_size[1], num_classes)
 
         self._init_layers()
 

src/otx/algo/common/utils/bbox_overlaps.py

@@ -8,6 +8,8 @@
 
 from __future__ import annotations
 
+import warnings
+
 import torch
 from torch import Tensor
 
@@ -142,15 +144,28 @@
         >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0)
         >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0)
     """
+    if not (bboxes1.size(-1) == 4 or bboxes1.size(0) == 0):
+        msg = "bboxes1 must have a last dimension of size 4 or be an empty tensor."
+        raise ValueError(msg)
+
+    if not (bboxes2.size(-1) == 4 or bboxes2.size(0) == 0):
+        msg = "bboxes2 must have a last dimension of size 4 or be an empty tensor."
+        raise ValueError(msg)
+
+    if bboxes1.shape[:-2] != bboxes2.shape[:-2]:
+        msg = "The batch dimension of bboxes must be the same."
+        raise ValueError(msg)
+
     batch_shape = bboxes1.shape[:-2]
 
     rows = bboxes1.size(-2)
     cols = bboxes2.size(-2)
 
     if rows * cols == 0:
+        warnings.warn("No bboxes are provided! Returning empty boxes!", stacklevel=2)
         if is_aligned:
-            return bboxes1.new((*batch_shape, rows))
-        return bboxes1.new((*batch_shape, rows, cols))
+            return bboxes1.new(batch_shape + (rows,))  # noqa: RUF005
+        return bboxes1.new(batch_shape + (rows, cols))  # noqa: RUF005
 
     area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1])
     area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1])

src/otx/algo/instance_segmentation/maskrcnn.py

@@ -327,7 +327,7 @@ def _exporter(self) -> OTXModelExporter:
                 "opset_version": 11,
                 "autograd_inlining": False,
             },
-            output_names=["bboxes", "labels", "masks"],
+            output_names=["bboxes", "labels", "masks", "feature_vector", "saliency_map"] if self.explain_mode else None,
         )
 
     def load_from_otx_v1_ckpt(self, state_dict: dict, add_prefix: str = "model.") -> dict:

src/otx/algo/instance_segmentation/rtmdet_inst.py

@@ -130,6 +130,7 @@ def _exporter(self) -> OTXModelExporter:
                 "opset_version": 11,
                 "autograd_inlining": False,
             },
+            # TODO(Eugene): Add XAI support for RTMDetInst
             output_names=["bboxes", "labels", "masks", "feature_vector", "saliency_map"] if self.explain_mode else None,
         )
 

src/otx/algo/instance_segmentation/segmentors/two_stage.py

@@ -226,19 +226,44 @@
         self,
         batch_inputs: torch.Tensor,
         batch_img_metas: list[dict],
-    ) -> tuple[torch.Tensor, ...]:
-        """Export for two stage detectors."""
-        x = self.extract_feat(batch_inputs)
+        explain_mode: bool,
+    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | dict:
+        """Export the model for ONNX/OpenVINO.
+
+        Args:
+            batch_inputs (torch.Tensor): image tensor with shape (N, C, H, W).
+            batch_img_metas (list[dict]): image information.
+            explain_mode (bool): whether to return feature vector.
 
+        Returns:
+            tuple[torch.Tensor, torch.Tensor, torch.Tensor] | dict:
+                - bboxes (torch.Tensor): bounding boxes.
+                - labels (torch.Tensor): labels.
+                - masks (torch.Tensor): masks.
+                - feature_vector (torch.Tensor, optional): feature vector.
+                - saliency_map (torch.Tensor, optional): saliency map.
+        """
+        x = self.extract_feat(batch_inputs)
         rpn_results_list = self.rpn_head.export(
             x,
             batch_img_metas,
             rescale=False,
         )
-
-        return self.roi_head.export(
+        bboxes, labels, masks = self.roi_head.export(
             x,
             rpn_results_list,
             batch_img_metas,
             rescale=False,
         )
+
+        if explain_mode:
+            feature_vector = self.feature_vector_fn(x)
+            return {
+                "bboxes": bboxes,
+                "labels": labels,
+                "masks": masks,
+                "feature_vector": feature_vector,
+                # create dummy tensor as model API supports saliency_map
+                "saliency_map": torch.zeros(1),
+            }
+        return bboxes, labels, masks