ENH: Report total, processed and failed items via finish_task

botcity/maestro/model.py

@@ -142,20 +142,27 @@ class AutomationTask:
     state: AutomationTaskState
     parameters: Dict[str, object]
     input_file: 'Artifact'
-    activity_id: int
-    activity_label: str
     agent_id: int
-    user_creation_id: int
+    user_email: str
     user_creation_name: str
-    org_creation_id: int
+    organization_label: str
     date_creation: str
     date_last_modified: str
     finish_status: AutomationTaskFinishStatus
     finish_message: str
     test: bool
+    machine_id: str
+    activity_label: str
     interrupted: bool
+    min_execution_date: str
     killed: bool
-    machine_id: str
+    date_start_running: str
+    priority: int
+    repository_label: str
+    processed_items: int
+    failed_items: int
+    total_items: int
+    activity_name: str
 
     def to_json(self) -> str:
         """
@@ -181,33 +188,45 @@ def from_json(payload: str) -> 'AutomationTask':
         uid = data.get("id")
         state = data.get("state")
         parameters = data.get("parameters")
-        activity_id = data.get("activityId")
-        activity_label = data.get("activityLabel")
         input_file = data.get("inputFile")
         if input_file:
             input_file = Artifact.from_dict(input_file)
         agent_id = data.get("agentId")
-        user_creation_id = data.get("userCreationId")
+        user_email = data.get("userEmail")
         user_creation_name = data.get("userCreationName")
-        org_creation_id = data.get("organizationCreationId")
+        organization_label = data.get("organizationLabel")
         date_creation = data.get("dateCreation")
         date_last_modified = data.get("dateLastModified")
         finish_status = data.get("finishStatus")
         finish_message = data.get("finishMessage")
-        machine_id = data.get("machineId")
         test = data.get("test", False)
+        machine_id = data.get("machineId")
+        activity_label = data.get("activityLabel")
         interrupted = data.get("interrupted", False)
         interrupted = False if interrupted is None else interrupted
+        min_execution_date = data.get("minExecutionDate")
         killed = data.get("killed", False)
         killed = False if killed is None else killed
-
-        return AutomationTask(id=uid, state=state, parameters=parameters, activity_id=activity_id,
+        date_start_running = data.get("dateStartRunning")
+        priority = data.get("priority")
+        repository_label = data.get("repositoryLabel")
+        processed_items = data.get("processedItems")
+        failed_items = data.get("failedItems")
+        total_items = data.get("totalItems")
+        activity_name = data.get("activityName")
+
+        return AutomationTask(id=uid, state=state, parameters=parameters,
                               activity_label=activity_label,
-                              input_file=input_file, agent_id=agent_id, user_creation_id=user_creation_id,
-                              user_creation_name=user_creation_name, org_creation_id=org_creation_id,
+                              input_file=input_file, agent_id=agent_id, user_email=user_email,
+                              user_creation_name=user_creation_name, organization_label=organization_label,
                               date_creation=date_creation, date_last_modified=date_last_modified,
-                              finish_status=finish_status, finish_message=finish_message, test=test,
-                              interrupted=interrupted, machine_id=machine_id, killed=killed)
+                              finish_status=finish_status, finish_message=finish_message,
+                              test=test, machine_id=machine_id,
+                              interrupted=interrupted, min_execution_date=min_execution_date, killed=killed,
+                              date_start_running=date_start_running, priority=priority,
+                              repository_label=repository_label,
+                              processed_items=processed_items, failed_items=failed_items, total_items=total_items,
+                              activity_name=activity_name)
 
     def is_interrupted(self) -> bool:
         """Whether or not this task received an interrupt request.

botcity/maestro/sdk.py

@@ -406,9 +406,33 @@ def create_task(self, activity_label: str, parameters: Dict[str, object],
                     message = 'Error during task create. Server returned %d. %s' % (req.status_code, req.text)
                 raise ValueError(message)
 
+    @staticmethod
+    def _validate_items(total_items, processed_items, failed_items):
+        if total_items == processed_items == failed_items is None:
+            # If all are None, return None for all
+            return None, None, None
+        if total_items is None and processed_items is not None and failed_items is not None:
+            # If total is None, but processed and failed are not, then total is the sum of both
+            total_items = processed_items + failed_items
+        if total_items is not None and processed_items is not None and failed_items is None:
+            # If total and processed are not None, but failed is, then failed is the difference
+            failed_items = total_items - processed_items
+        if total_items is not None and processed_items is None and failed_items is not None:
+            # If total and failed are not None, but processed is, then processed is the difference
+            processed_items = total_items - failed_items
+        # Make sure no negative values are present
+        total_items = max(0, total_items)
+        processed_items = max(0, processed_items)
+        failed_items = max(0, failed_items)
+        if total_items is not None and processed_items is not None and failed_items is not None:
+            if total_items != processed_items + failed_items:
+                raise ValueError("Total items is not equal to the sum of processed and failed items.")
+        return total_items, processed_items, failed_items
+
     @ensure_access_token()
     def finish_task(self, task_id: str, status: model.AutomationTaskFinishStatus,
-                    message: str = "") -> model.ServerMessage:
+                    message: str = "", total_items: int = None, processed_items: int = None,
+                    failed_items: int = None) -> model.ServerMessage:
         """
         Finishes a given task.
 
@@ -417,13 +441,31 @@ def finish_task(self, task_id: str, status: model.AutomationTaskFinishStatus,
             status: The condition in which the task must be finished.
                 See [AutomationTaskFinishStatus][botcity.maestro.model.AutomationTaskFinishStatus]
             message: A message to be associated with this action.
+            total_items: Total number of items processed by the task.
+            processed_items: Number items processed successfully by the task.
+            failed_items: Number items failed to be processed by the task.
+
+        Note:
+            Starting from version 0.5.0, the parameters `total_items`, `processed_items` and `failed_items` are
+            available to be used. It is important to report the correct values for these parameters as they are used
+            to calculate the ROI, success rate and other metrics.
+
+            Keep in mind that the sum of `processed_items` and `failed_items` must be equal to `total_items`. If
+            `total_items` is None, then the sum of `processed_items` and `failed_items` will be used as `total_items`.
+            If you inform `total_items` and `processed_items`, then `failed_items` will be calculated as the difference.
+
 
         Returns:
             Server response message. See [ServerMessage][botcity.maestro.model.ServerMessage]
         """
         url = f'{self._server}/api/v2/task/{task_id}'
+
+        total_items, processed_items, failed_items = self._validate_items(total_items, processed_items, failed_items)
+
         data = {"finishStatus": status, "finishMessage": message,
-                "state": "FINISHED"}
+                "state": "FINISHED", "totalItems": total_items,
+                "processedItems": processed_items, "failedItems": failed_items}
+
         headers = self._headers()
         with requests.post(url, json=data, headers=headers, timeout=self._timeout, verify=self.VERIFY_SSL_CERT) as req:
             if req.ok:

conftest.py

@@ -9,9 +9,12 @@
 from uuid import uuid4
 
 import pytest
+import dotenv
 
 from botcity.maestro import *
 
+dotenv.load_dotenv()
+
 SERVER = os.getenv("BOTCITY_SERVER")
 LOGIN = os.getenv("BOTCITY_LOGIN")
 KEY = os.getenv("BOTCITY_KEY")
@@ -73,7 +76,7 @@ def pool(activity_label: str, maestro: BotMaestroSDK) -> DataPool:
     pool._delete()
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture()
 def task(maestro: BotMaestroSDK, activity_label: str):
     parameters = {
         "test_to_test": "testing",

test-requirements.txt

@@ -3,4 +3,5 @@ mypy
 flake8
 isort
 pre-commit
-pytest-depends
+pytest-depends
+python-dotenv

tests/integration/test_artifact.py

@@ -17,13 +17,13 @@ def test_post_artifact(maestro: BotMaestroSDK, file: str, task: AutomationTask):
 
 @pytest.mark.depends(name="test_post_artifact")
 def test_list_artifacts(maestro: BotMaestroSDK):
-    list_artifact = maestro.list_artifacts()
+    list_artifact = maestro.list_artifacts(days=1)
     assert len(list_artifact) > 0
 
 
 @pytest.mark.depends(name="test_list_artifacts")
 def test_get_artifact(maestro: BotMaestroSDK, tmp_folder: str):
-    list_artifact = maestro.list_artifacts()
+    list_artifact = maestro.list_artifacts(days=1)
     name, content = maestro.get_artifact(artifact_id=list_artifact[0].id)
     filepath = f"{tmp_folder}/{name}"
 

tests/integration/test_task.py

@@ -2,6 +2,8 @@
 import datetime
 from random import randint
 
+import pytest
+
 from botcity.maestro import (AutomationTask, AutomationTaskFinishStatus,
                              BotMaestroSDK)
 
@@ -55,3 +57,87 @@ def test_finish_task_to_failed(maestro: BotMaestroSDK, task: AutomationTask):
     )
     task = maestro.get_task(task_id=str(task.id))
     assert task.finish_status == AutomationTaskFinishStatus.FAILED
+
+def test_finish_task_report_no_items(maestro: BotMaestroSDK, task: AutomationTask):
+    maestro.finish_task(
+        task_id=str(task.id),
+        message="Task Finished OK.",
+        status=AutomationTaskFinishStatus.SUCCESS,
+    )
+    task = maestro.get_task(task_id=str(task.id))
+    assert task.finish_status == AutomationTaskFinishStatus.SUCCESS
+    assert task.total_items == 0
+    assert task.processed_items == 0
+    assert task.failed_items == 0
+
+def test_finish_task_report_items(maestro: BotMaestroSDK, task: AutomationTask):
+    maestro.finish_task(
+        task_id=str(task.id),
+        message="Task Finished with success.",
+        status=AutomationTaskFinishStatus.SUCCESS,
+        total_items=10,
+        processed_items=5,
+        failed_items=5
+    )
+    task = maestro.get_task(task_id=str(task.id))
+    assert task.finish_status == AutomationTaskFinishStatus.SUCCESS
+    assert task.total_items == 10
+    assert task.processed_items == 5
+    assert task.failed_items == 5
+
+
+def test_finish_task_report_total_and_processed(maestro: BotMaestroSDK, task: AutomationTask):
+    maestro.finish_task(
+        task_id=str(task.id),
+        message="Task Finished with success.",
+        status=AutomationTaskFinishStatus.SUCCESS,
+        total_items=10,
+        processed_items=5
+    )
+    task = maestro.get_task(task_id=str(task.id))
+    assert task.finish_status == AutomationTaskFinishStatus.SUCCESS
+    assert task.total_items == 10
+    assert task.processed_items == 5
+    assert task.failed_items == 5
+
+
+def test_finish_task_report_total_and_failed(maestro: BotMaestroSDK, task: AutomationTask):
+    maestro.finish_task(
+        task_id=str(task.id),
+        message="Task Finished with success.",
+        status=AutomationTaskFinishStatus.SUCCESS,
+        total_items=10,
+        failed_items=5
+    )
+    task = maestro.get_task(task_id=str(task.id))
+    assert task.finish_status == AutomationTaskFinishStatus.SUCCESS
+    assert task.total_items == 10
+    assert task.processed_items == 5
+    assert task.failed_items == 5
+
+
+def test_finish_task_report_processed_and_failed(maestro: BotMaestroSDK, task: AutomationTask):
+    maestro.finish_task(
+        task_id=str(task.id),
+        message="Task Finished with success.",
+        status=AutomationTaskFinishStatus.SUCCESS,
+        processed_items=5,
+        failed_items=5
+    )
+    task = maestro.get_task(task_id=str(task.id))
+    assert task.finish_status == AutomationTaskFinishStatus.SUCCESS
+    assert task.total_items == 10
+    assert task.processed_items == 5
+    assert task.failed_items == 5
+
+
+def test_finish_task_report_error_invalid_total_items(maestro: BotMaestroSDK, task: AutomationTask):
+    with pytest.raises(ValueError):
+        maestro.finish_task(
+            task_id=str(task.id),
+            message="Task Finished with success.",
+            status=AutomationTaskFinishStatus.SUCCESS,
+            total_items=10,
+            processed_items=6,
+            failed_items=5
+        )

tests/model/test_task.py

@@ -0,0 +1,42 @@
+import pytest
+from botcity.maestro import BotMaestroSDK
+
+def test_items_all_none():
+    sdk = BotMaestroSDK()
+    total, processed, failed = sdk._validate_items(None, None, None)
+    assert total is None
+    assert processed is None
+    assert failed is None
+
+def test_items_total_none():
+    sdk = BotMaestroSDK()
+    total, processed, failed = sdk._validate_items(None, 5, 5)
+    assert total == 10
+    assert processed == 5
+    assert failed == 5
+
+def test_items_failed_none():
+    sdk = BotMaestroSDK()
+    total, processed, failed = sdk._validate_items(10, 5, None)
+    assert total == 10
+    assert processed == 5
+    assert failed == 5
+
+def test_items_processed_none():
+    sdk = BotMaestroSDK()
+    total, processed, failed = sdk._validate_items(10, None, 5)
+    assert total == 10
+    assert processed == 5
+    assert failed == 5
+
+def test_items_no_negative_values():
+    sdk = BotMaestroSDK()
+    total, processed, failed = sdk._validate_items(-10, -5, -5)
+    assert total == 0
+    assert processed == 0
+    assert failed == 0
+
+def test_items_total_not_equal_to_sum():
+    sdk = BotMaestroSDK()
+    with pytest.raises(ValueError):
+        sdk._validate_items(10, 5, 6)