[PLT-1614] Support data row / batch for live mmc projects (#1856)

Author: vbrodsky

docs/labelbox/datarow_payload_templates.rst

@@ -0,0 +1,6 @@
+Datarow payload templates
+===============================================================================================
+
+.. automodule:: labelbox.schema.data_row_payload_templates
+   :members:
+   :show-inheritance:

libs/labelbox/src/labelbox/client.py

@@ -6,6 +6,7 @@
 import random
 import time
 import urllib.parse
+import warnings
 from collections import defaultdict
 from datetime import datetime, timezone
 from types import MappingProxyType
@@ -637,6 +638,7 @@ def create_project(
         }
         return self._create_project(_CoreProjectInput(**input))
 
+    @overload
     def create_model_evaluation_project(
         self,
         name: str,
@@ -649,7 +651,17 @@ def create_model_evaluation_project(
         is_consensus_enabled: Optional[bool] = None,
         dataset_id: Optional[str] = None,
         dataset_name: Optional[str] = None,
-        data_row_count: int = 100,
+        data_row_count: Optional[int] = None,
+        **kwargs,
+    ) -> Project:
+        pass
+
+    def create_model_evaluation_project(
+        self,
+        dataset_id: Optional[str] = None,
+        dataset_name: Optional[str] = None,
+        data_row_count: Optional[int] = None,
+        **kwargs,
     ) -> Project:
         """
         Use this method exclusively to create a chat model evaluation project.
@@ -674,22 +686,39 @@ def create_model_evaluation_project(
             >>> client.create_model_evaluation_project(name=project_name, dataset_id="clr00u8j0j0j0", data_row_count=10)
             >>>     This creates a new project, and adds 100 datarows to the dataset with id "clr00u8j0j0j0" and assigns a batch of the newly created 10 data rows to the project.
 
+            >>> client.create_model_evaluation_project(name=project_name)
+            >>>     This creates a new project with no data rows.
 
         """
-        if not dataset_id and not dataset_name:
-            raise ValueError(
-                "dataset_name or data_set_id must be present and not be an empty string."
-            )
+        autogenerate_data_rows = False
+        dataset_name_or_id = None
+        append_to_existing_dataset = None
+
+        if dataset_id or dataset_name:
+            autogenerate_data_rows = True
 
         if dataset_id:
             append_to_existing_dataset = True
             dataset_name_or_id = dataset_id
-        else:
+        elif dataset_name:
             append_to_existing_dataset = False
             dataset_name_or_id = dataset_name
 
-        media_type = MediaType.Conversational
-        editor_task_type = EditorTaskType.ModelChatEvaluation
+        if autogenerate_data_rows:
+            kwargs["dataset_name_or_id"] = dataset_name_or_id
+            kwargs["append_to_existing_dataset"] = append_to_existing_dataset
+            if data_row_count is None:
+                data_row_count = 100
+            if data_row_count < 0:
+                raise ValueError("data_row_count must be a positive integer.")
+            kwargs["data_row_count"] = data_row_count
+            warnings.warn(
+                "Automatic generation of data rows of live model evaluation projects is deprecated. dataset_name_or_id, append_to_existing_dataset, data_row_count will be removed in a future version.",
+                DeprecationWarning,
+            )
+
+        kwargs["media_type"] = MediaType.Conversational
+        kwargs["editor_task_type"] = EditorTaskType.ModelChatEvaluation.value
 
         input = {
             "name": name,

libs/labelbox/src/labelbox/data/annotation_types/collection.py

@@ -1,14 +1,10 @@
 import logging
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Callable, Generator, Iterable, Union, Optional
-from uuid import uuid4
 import warnings
+from typing import Callable, Generator, Iterable, Union
 
-from tqdm import tqdm
-
-from labelbox.schema import ontology
 from labelbox.orm.model import Entity
-from ..ontology import get_classifications, get_tools
+from labelbox.schema import ontology
+
 from ..generator import PrefetchGenerator
 from .label import Label
 

libs/labelbox/src/labelbox/schema/data_row_payload_templates.py

@@ -0,0 +1,40 @@
+from typing import Dict, List
+
+from pydantic import BaseModel, Field
+
+from labelbox.schema.data_row import DataRowMetadataField
+
+
+class ModelEvalutationTemlateRowData(BaseModel):
+    type: str = Field(
+        default="application/vnd.labelbox.conversational.model-chat-evaluation",
+        frozen=True,
+    )
+    draft: bool = Field(default=True, frozen=True)
+    rootMessageIds: List[str] = Field(default=[])
+    actors: Dict = Field(default={})
+    version: int = Field(default=2, frozen=True)
+    messages: Dict = Field(default={})
+
+
+class ModelEvaluationTemplate(BaseModel):
+    """
+    Use this class to create a model evaluation data row.
+
+    Examples:
+        >>> data = ModelEvaluationTemplate()
+        >>> data.row_data.rootMessageIds = ["root1"]
+        >>> vector = [random.uniform(1.0, 2.0) for _ in range(embedding.dims)]
+        >>> data.embeddings = [...]
+        >>> data.metadata_fields = [...]
+        >>> data.attachments = [...]
+        >>> content = data.model_dump()
+        >>> task = dataset.create_data_rows([content])
+    """
+
+    row_data: ModelEvalutationTemlateRowData = Field(
+        default=ModelEvalutationTemlateRowData()
+    )
+    attachments: List[Dict] = Field(default=[])
+    embeddings: List[Dict] = Field(default=[])
+    metadata_fields: List[DataRowMetadataField] = Field(default=[])

libs/labelbox/src/labelbox/schema/project.py

@@ -13,7 +13,6 @@
     List,
     Optional,
     Tuple,
-    TypeVar,
     Union,
     get_args,
 )
@@ -23,7 +22,6 @@
     InvalidQueryError,
     LabelboxError,
     ProcessingWaitTimeout,
-    ResourceConflict,
     ResourceNotFoundError,
     error_message_for_unparsed_graphql_error,
 )  # type: ignore
@@ -59,7 +57,6 @@
 from labelbox.schema.model_config import ModelConfig
 from labelbox.schema.ontology_kind import (
     EditorTaskType,
-    OntologyKind,
     UploadType,
 )
 from labelbox.schema.project_model_config import ProjectModelConfig
@@ -577,7 +574,7 @@ def upsert_instructions(self, instructions_file: str) -> None:
 
         if frontend.name != "Editor":
             logger.warning(
-                f"This function has only been tested to work with the Editor front end. Found %s",
+                "This function has only been tested to work with the Editor front end. Found %s",
                 frontend.name,
             )
 
@@ -745,7 +742,9 @@ def create_batch(
             lbox.exceptions.ValueError if a project is not batch mode, if the project is auto data generation, if the batch exceeds 100k data rows
         """
 
-        if self.is_auto_data_generation():
+        if (
+            self.is_auto_data_generation() and not self.is_chat_evaluation()
+        ):  # NOTE live chat evaluatiuon projects in sdk do not pre-generate data rows, but use batch as all other projects
             raise ValueError(
                 "Cannot create batches for auto data generation projects"
             )
@@ -771,7 +770,7 @@ def create_batch(
 
         if row_count > 100_000:
             raise ValueError(
-                f"Batch exceeds max size, break into smaller batches"
+                "Batch exceeds max size, break into smaller batches"
             )
         if not row_count:
             raise ValueError("You need at least one data row in a batch")
@@ -1039,8 +1038,7 @@ def _create_batch_async(
         task = self._wait_for_task(task_id)
         if task.status != "COMPLETE":
             raise LabelboxError(
-                f"Batch was not created successfully: "
-                + json.dumps(task.errors)
+                "Batch was not created successfully: " + json.dumps(task.errors)
             )
 
         return self.client.get_batch(self.uid, batch_id)
@@ -1262,7 +1260,7 @@ def update_data_row_labeling_priority(
         task = self._wait_for_task(task_id)
         if task.status != "COMPLETE":
             raise LabelboxError(
-                f"Priority was not updated successfully: "
+                "Priority was not updated successfully: "
                 + json.dumps(task.errors)
             )
         return True
@@ -1442,7 +1440,7 @@ def move_data_rows_to_task_queue(
         task = self._wait_for_task(task_id)
         if task.status != "COMPLETE":
             raise LabelboxError(
-                f"Data rows were not moved successfully: "
+                "Data rows were not moved successfully: "
                 + json.dumps(task.errors)
             )
 

libs/labelbox/tests/integration/conftest.py

@@ -631,11 +631,28 @@ def chat_evaluation_ontology(client, rand_gen):
 
 
 @pytest.fixture
-def live_chat_evaluation_project_with_new_dataset(client, rand_gen):
+def live_chat_evaluation_project(client, rand_gen):
     project_name = f"test-model-evaluation-project-{rand_gen(str)}"
-    dataset_name = f"test-model-evaluation-dataset-{rand_gen(str)}"
-    project = client.create_model_evaluation_project(
-        name=project_name, dataset_name=dataset_name, data_row_count=1
+    project = client.create_model_evaluation_project(name=project_name)
+
+    yield project
+
+    project.delete()
+
+
+@pytest.fixture
+def live_chat_evaluation_project_with_batch(
+    client,
+    rand_gen,
+    live_chat_evaluation_project,
+    offline_conversational_data_row,
+):
+    project_name = f"test-model-evaluation-project-{rand_gen(str)}"
+    project = client.create_model_evaluation_project(name=project_name)
+
+    project.create_batch(
+        rand_gen(str),
+        [offline_conversational_data_row.uid],  # sample of data row objects
     )
 
     yield project

libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py

@@ -1,15 +1,12 @@
-from unittest.mock import patch
-
 import pytest
 
 from labelbox import MediaType
 from labelbox.schema.ontology_kind import OntologyKind
 
 
 def test_create_chat_evaluation_ontology_project(
-    client,
     chat_evaluation_ontology,
-    live_chat_evaluation_project_with_new_dataset,
+    live_chat_evaluation_project,
     offline_conversational_data_row,
     rand_gen,
 ):
@@ -28,36 +25,19 @@ def test_create_chat_evaluation_ontology_project(
         assert classification.schema_id
         assert classification.feature_schema_id
 
-    project = live_chat_evaluation_project_with_new_dataset
+    project = live_chat_evaluation_project
     assert project.model_setup_complete is None
 
     project.connect_ontology(ontology)
 
     assert project.labeling_frontend().name == "Editor"
     assert project.ontology().name == ontology.name
 
-    with pytest.raises(
-        ValueError,
-        match="Cannot create batches for auto data generation projects",
-    ):
-        project.create_batch(
-            rand_gen(str),
-            [offline_conversational_data_row.uid],  # sample of data row objects
-        )
-
-    with pytest.raises(
-        ValueError,
-        match="Cannot create batches for auto data generation projects",
-    ):
-        with patch(
-            "labelbox.schema.project.MAX_SYNC_BATCH_ROW_COUNT", new=0
-        ):  # force to async
-            project.create_batch(
-                rand_gen(str),
-                [
-                    offline_conversational_data_row.uid
-                ],  # sample of data row objects
-            )
+    batch = project.create_batch(
+        rand_gen(str),
+        [offline_conversational_data_row.uid],  # sample of data row objects
+    )
+    assert batch
 
 
 def test_create_chat_evaluation_ontology_project_existing_dataset(

libs/labelbox/tests/integration/test_data_rows.py

@@ -405,7 +405,7 @@ def test_create_data_row_with_metadata_dict(
         row_data=image_url, metadata_fields=make_metadata_fields_dict
     )
 
-    assert len(list(dataset.data_rows())) == 1
+    assert len([dr for dr in dataset.data_rows()]) == 1
     assert data_row.dataset() == dataset
     assert data_row.created_by() == client.get_user()
     assert data_row.organization() == client.get_organization()

libs/labelbox/tests/integration/test_labeling_service.py

@@ -43,12 +43,11 @@ def test_request_labeling_service_moe_offline_project(
 
 
 def test_request_labeling_service_moe_project(
-    rand_gen,
-    live_chat_evaluation_project_with_new_dataset,
+    live_chat_evaluation_project_with_batch,
     chat_evaluation_ontology,
     model_config,
 ):
-    project = live_chat_evaluation_project_with_new_dataset
+    project = live_chat_evaluation_project_with_batch
     project.connect_ontology(chat_evaluation_ontology)
 
     project.upsert_instructions("tests/integration/media/sample_pdf.pdf")