[FIX] Fix label mismatch of evaluation and validation with large dataset in semantic segmentation

otx/algorithms/segmentation/adapters/mmseg/data/dataset.py

@@ -127,7 +127,7 @@ def __init__(
         self.label_map = None
 
         dataset_labels = self.otx_dataset.get_labels(include_empty=False)
-        self.project_labels = self.filter_labels(dataset_labels, classes)
+        self.project_labels = sorted(self.filter_labels(dataset_labels, classes))
         self.CLASSES, self.PALETTE = self.get_classes_and_palette(classes, None)
 
         # Instead of using list data_infos as in CustomDataset, this implementation of dataset
@@ -144,8 +144,8 @@ def __init__(
 
     @staticmethod
     @check_input_parameters_type()
-    def filter_labels(all_labels: List[LabelEntity], label_names: List[str]):
-        """Filtering Labels function."""
+    def filter_labels(all_labels: List[LabelEntity], label_names: List[str]) -> List[LabelEntity]:
+        """Filter and collect actual label entities."""
         filtered_labels = []
         for label_name in label_names:
             matches = [label for label in all_labels if label.name == label_name]
@@ -263,6 +263,11 @@ def __init__(self, **kwargs):
         else:
             classes = []
         super().__init__(otx_dataset=otx_dataset, pipeline=pipeline, classes=classes)
+
+        self.CLASSES = [label.name for label in self.project_labels]
+        if "background" not in self.CLASSES:
+            self.CLASSES = ["background"] + self.CLASSES
+
         if self.label_map is None:
             self.label_map = {}
             for i, c in enumerate(self.CLASSES):

otx/algorithms/segmentation/tasks/inference.py

@@ -93,7 +93,7 @@ def __init__(self, task_environment: TaskEnvironment, **kwargs):
         self._label_dictionary = {}  # type: Dict
 
         super().__init__(SegmentationConfig, task_environment, **kwargs)
-        self._label_dictionary = dict(enumerate(self._labels, 1))
+        self._label_dictionary = dict(enumerate(sorted(self._labels), 1))
 
     @check_input_parameters_type({"dataset": DatasetParamTypeCheck})
     def infer(

otx/api/usecases/evaluation/basic_operations.py

@@ -34,6 +34,8 @@ def get_intersections_and_cardinalities(
     Returns:
         Tuple[NumberPerLabel, NumberPerLabel]: (all_intersections, all_cardinalities)
     """
+
+    # TODO [Soobee] : Add score for background label and align the calculation method with validation
     all_intersections: NumberPerLabel = {label: 0 for label in labels}
     all_intersections[None] = 0
     all_cardinalities: NumberPerLabel = {label: 0 for label in labels}

tests/unit/algorithms/segmentation/adapters/mmseg/test_dataset.py

@@ -13,15 +13,16 @@
 )
 from otx.api.entities.dataset_item import DatasetItemEntity
 from otx.api.entities.datasets import DatasetEntity
+from otx.api.entities.id import ID
 from otx.api.entities.image import Image
 from otx.api.entities.label import Domain, LabelEntity
 from otx.api.entities.scored_label import ScoredLabel
 from otx.api.entities.shapes.rectangle import Rectangle
 from tests.test_suite.e2e_test_system import e2e_pytest_unit
 
 
-def label_entity(name="test label") -> LabelEntity:
-    return LabelEntity(name=name, domain=Domain.SEGMENTATION)
+def label_entity(name="test label", id="0") -> LabelEntity:
+    return LabelEntity(name=name, id=ID(id), domain=Domain.SEGMENTATION)
 
 
 def dataset_item() -> DatasetItemEntity:
@@ -35,15 +36,18 @@ def dataset_item() -> DatasetItemEntity:
 
 class TestMPASegDataset:
     @pytest.fixture(autouse=True)
-    def setUp(self) -> None:
+    def setUp(self, mocker) -> None:
         self.otx_dataset: DatasetEntity = DatasetEntity(items=[dataset_item()])
         self.pipeline: list[dict] = [{"type": "LoadImageFromOTXDataset", "to_float32": True}]
         self.classes: list[str] = ["class_1", "class_2"]
+        labels_entities = [label_entity(name, i) for i, name in enumerate(self.classes)]
+
+        mocker.patch.object(MPASegDataset, "filter_labels", return_value=labels_entities)
 
         self.dataset: MPASegDataset = MPASegDataset(
             otx_dataset=self.otx_dataset,
             pipeline=self.pipeline,
-            labels=[label_entity(name) for name in self.classes],
+            labels=labels_entities,
             new_classes=self.classes,
         )
 
@@ -58,6 +62,27 @@ def test_mpasegdataset_initialization(self) -> None:
         # Check if label_map is created as expected
         assert self.dataset.label_map == {0: 0, 1: 1, 2: 2}
 
+    @e2e_pytest_unit
+    def test_classes_sorted(self, mocker) -> None:
+        self.otx_dataset: DatasetEntity = DatasetEntity(items=[dataset_item()])
+        self.pipeline: list[dict] = [{"type": "LoadImageFromOTXDataset", "to_float32": True}]
+        self.classes: list[str] = [f"class_{i+1}" for i in range(11)]
+        labels_entities = [label_entity(name, i) for i, name in enumerate(self.classes)]
+
+        mocker.patch.object(MPASegDataset, "filter_labels", return_value=labels_entities)
+
+        self.dataset: MPASegDataset = MPASegDataset(
+            otx_dataset=self.otx_dataset,
+            pipeline=self.pipeline,
+            labels=labels_entities,
+            new_classes=self.classes,
+        )
+
+        assert self.dataset.CLASSES != ["background"] + self.classes
+        assert self.dataset.CLASSES == ["background"] + [label.name for label in sorted(labels_entities)]
+
+        assert self.dataset.label_map == {0: 0, 1: 1, 2: 2, 3: 11, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8, 10: 9, 11: 10}
+
     @e2e_pytest_unit
     def test_getitem_method(self) -> None:
         data_item: dict = self.dataset[0]