Introduce Active Learning

docker/config/cpu_http.py

@@ -1,4 +1,6 @@
+from inference.core.cache import cache
 from inference.core.interfaces.http.http_api import HttpInterface
+from inference.core.managers.active_learning import ActiveLearningManager, BackgroundTaskActiveLearningManager
 from inference.core.managers.base import ModelManager
 from inference.core.managers.decorators.fixed_size_cache import WithFixedSizeCache
 from inference.core.registries.roboflow import (
@@ -7,17 +9,27 @@
 import os
 from prometheus_fastapi_instrumentator import Instrumentator
 
-from inference.core.env import MAX_ACTIVE_MODELS
+from inference.core.env import MAX_ACTIVE_MODELS, ACTIVE_LEARNING_ENABLED, LAMBDA
 from inference.models.utils import ROBOFLOW_MODEL_TYPES
 
 model_registry = RoboflowModelRegistry(ROBOFLOW_MODEL_TYPES)
+
+if ACTIVE_LEARNING_ENABLED:
+    if LAMBDA:
+        model_manager = ActiveLearningManager(model_registry=model_registry, cache=cache)
+    else:
+        model_manager = BackgroundTaskActiveLearningManager(model_registry=model_registry, cache=cache)
+else:
+    model_manager = ModelManager(model_registry=model_registry)
+
 model_manager = WithFixedSizeCache(
-    ModelManager(model_registry), max_size=MAX_ACTIVE_MODELS
+    model_manager,
+    max_size=MAX_ACTIVE_MODELS
 )
+
 model_manager.model_manager.init_pingback()
 interface = HttpInterface(model_manager)
 app = interface.app
-
 # Setup Prometheus scraping endpoint at /metrics
 # More info: https://github.com/trallnag/prometheus-fastapi-instrumentator
 if os.environ.get("ENABLE_PROMETHEUS", False):

docker/config/trt_http.py

@@ -1,8 +1,10 @@
 import os
 from prometheus_fastapi_instrumentator import Instrumentator
 
-from inference.core.env import MAX_ACTIVE_MODELS
+from inference.core.cache import cache
+from inference.core.env import MAX_ACTIVE_MODELS, ACTIVE_LEARNING_ENABLED, LAMBDA
 from inference.core.interfaces.http.http_api import HttpInterface
+from inference.core.managers.active_learning import ActiveLearningManager
 from inference.core.managers.base import ModelManager
 from inference.core.managers.decorators.fixed_size_cache import WithFixedSizeCache
 from inference.core.registries.roboflow import (
@@ -12,8 +14,17 @@
 
 
 model_registry = RoboflowModelRegistry(ROBOFLOW_MODEL_TYPES)
+
+if ACTIVE_LEARNING_ENABLED:
+    if LAMBDA:
+        model_manager = ActiveLearningManager(model_registry=model_registry, cache=cache)
+    else:
+        model_manager = ActiveLearningManager(model_registry=model_registry, cache=cache)
+else:
+    model_manager = ModelManager(model_registry=model_registry)
+
 model_manager = WithFixedSizeCache(
-    ModelManager(model_registry), max_size=MAX_ACTIVE_MODELS
+    model_manager, max_size=MAX_ACTIVE_MODELS
 )
 model_manager.model_manager.init_pingback()
 interface = HttpInterface(

inference/core/active_learning/accounting.py

@@ -0,0 +1,63 @@
+from typing import List, Optional
+
+from inference.core.entities.types import DatasetID, WorkspaceID
+from inference.core.roboflow_api import (
+    get_roboflow_labeling_batches,
+    get_roboflow_labeling_jobs,
+)
+
+
+def image_can_be_submitted_to_batch(
+    batch_name: str,
+    workspace_id: WorkspaceID,
+    dataset_id: DatasetID,
+    max_batch_images: Optional[int],
+    api_key: str,
+) -> bool:
+    if max_batch_images is None:
+        return True
+    labeling_batches = get_roboflow_labeling_batches(
+        api_key=api_key,
+        workspace_id=workspace_id,
+        dataset_id=dataset_id,
+    )
+    matching_labeling_batch = get_matching_labeling_batch(
+        all_labeling_batches=labeling_batches["batches"],
+        batch_name=batch_name,
+    )
+    if matching_labeling_batch is None:
+        return max_batch_images > 0
+    batch_images_under_labeling = 0
+    if matching_labeling_batch["numJobs"] > 0:
+        labeling_jobs = get_roboflow_labeling_jobs(
+            api_key=api_key, workspace_id=workspace_id, dataset_id=dataset_id
+        )
+        batch_images_under_labeling = get_images_in_labeling_jobs_of_specific_batch(
+            all_labeling_jobs=labeling_jobs["jobs"],
+            batch_id=matching_labeling_batch["id"],
+        )
+    total_batch_images = matching_labeling_batch["images"] + batch_images_under_labeling
+    return max_batch_images > total_batch_images
+
+
+def get_matching_labeling_batch(
+    all_labeling_batches: List[dict],
+    batch_name: str,
+) -> Optional[dict]:
+    matching_batch = None
+    for labeling_batch in all_labeling_batches:
+        if labeling_batch["name"] == batch_name:
+            matching_batch = labeling_batch
+            break
+    return matching_batch
+
+
+def get_images_in_labeling_jobs_of_specific_batch(
+    all_labeling_jobs: List[dict],
+    batch_id: str,
+) -> int:
+    matching_jobs = []
+    for labeling_job in all_labeling_jobs:
+        if batch_id in labeling_job["sourceBatch"]:
+            matching_jobs.append(labeling_job)
+    return sum(job["numImages"] for job in matching_jobs)

inference/core/active_learning/batching.py

@@ -0,0 +1,26 @@
+from inference.core.active_learning.entities import (
+    ActiveLearningConfiguration,
+    BatchReCreationInterval,
+)
+from inference.core.active_learning.utils import (
+    generate_start_timestamp_for_this_month,
+    generate_start_timestamp_for_this_week,
+    generate_today_timestamp,
+)
+
+RECREATION_INTERVAL2TIMESTAMP_GENERATOR = {
+    BatchReCreationInterval.DAILY: generate_today_timestamp,
+    BatchReCreationInterval.WEEKLY: generate_start_timestamp_for_this_week,
+    BatchReCreationInterval.MONTHLY: generate_start_timestamp_for_this_month,
+}
+
+
+def generate_batch_name(configuration: ActiveLearningConfiguration) -> str:
+    batch_name = configuration.batches_name_prefix
+    if configuration.batch_recreation_interval is BatchReCreationInterval.NEVER:
+        return batch_name
+    timestamp_generator = RECREATION_INTERVAL2TIMESTAMP_GENERATOR[
+        configuration.batch_recreation_interval
+    ]
+    timestamp = timestamp_generator()
+    return f"{batch_name}_{timestamp}"