Optimized jobs request (cvat-ai#5962)

retailnext · Jul 1, 2023 · 8834a78 · 8834a78
1 parent 6af2ebf
commit 8834a78
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 26 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -50,8 +50,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (<https://github.com/opencv/cvat/issues/5566>)
 - Rotation property lost during saving a mutable attribute (<https://github.com/opencv/cvat/pull/5968>)
 <<<<<<< HEAD
+<<<<<<< HEAD
 - Incorrect calculation of working time in analytics (<https://github.com/opencv/cvat/pull/5973>)
 =======
+=======
+- Optimized /api/jobs request (<https://github.com/opencv/cvat/pull/5962>)
+>>>>>>> 14dc68e7d (Optimized jobs request (#5962))
 - Server micro version support check in SDK/CLI (<https://github.com/opencv/cvat/pull/5991>)
 <<<<<<< HEAD
 >>>>>>> 0c853b15b (Fix version checks in SDK (#5991))

diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py
@@ -136,12 +136,6 @@ def get_fields(self):
     def get_attribute(self, instance):
         return instance
 
-
-class LabelsSummarySerializer(_CollectionSummarySerializer):
-    def __init__(self, *, model=models.Label, url_filter_key, source='get_labels', **kwargs):
-        super().__init__(model=model, url_filter_key=url_filter_key, source=source, **kwargs)
-
-
 class JobsSummarySerializer(_CollectionSummarySerializer):
     completed = serializers.IntegerField(source='completed_jobs_count', default=0)
 
@@ -156,10 +150,36 @@ class TasksSummarySerializer(_CollectionSummarySerializer):
 class CommentsSummarySerializer(_CollectionSummarySerializer):
     pass
 
+class BasicSummarySerializer(serializers.Serializer):
+    url = serializers.URLField(read_only=True)
+    count = serializers.IntegerField(read_only=True)
 
-class IssuesSummarySerializer(_CollectionSummarySerializer):
-    pass
+    def to_representation(self, instance):
+        request = self.context.get('request')
+        if not request:
+            return None
 
+        return {
+            'url': self.get_url(request, instance),
+            'count': self.get_count(instance)
+        }
+
+class LabelsSummarySerializer(BasicSummarySerializer):
+    def get_url(self, request, instance):
+        filter_key = instance.__class__.__name__.lower() + '_id'
+        return reverse('label-list', request=request,
+            query_params={ filter_key: instance.id })
+
+    def get_count(self, instance):
+        return getattr(instance, 'task_labels_count', 0) + getattr(instance, 'proj_labels_count', 0)
+
+class IssuesSummarySerializer(BasicSummarySerializer):
+    def get_url(self, request, instance):
+        return reverse('issue-list', request=request,
+            query_params={ 'job_id': instance.id })
+
+    def get_count(self, instance):
+        return getattr(instance, 'issues__count', 0)
 
 class BasicUserSerializer(serializers.ModelSerializer):
     def validate(self, attrs):
@@ -532,15 +552,16 @@ class JobReadSerializer(serializers.ModelSerializer):
     mode = serializers.ReadOnlyField(source='segment.task.mode')
     bug_tracker = serializers.CharField(max_length=2000, source='get_bug_tracker',
         allow_null=True, read_only=True)
-    labels = LabelsSummarySerializer(url_filter_key='job_id')
-    issues = IssuesSummarySerializer(models.Issue, url_filter_key='job_id')
+    labels = LabelsSummarySerializer(source='*')
+    issues = IssuesSummarySerializer(source='*')
 
     class Meta:
         model = models.Job
         fields = ('url', 'id', 'task_id', 'project_id', 'assignee',
             'dimension', 'bug_tracker', 'status', 'stage', 'state', 'mode',
             'start_frame', 'stop_frame', 'data_chunk_size', 'data_compressed_chunk_type',
-            'updated_date', 'issues', 'labels')
+            'updated_date', 'issues', 'labels'
+        )
         read_only_fields = fields
 
 class JobWriteSerializer(serializers.ModelSerializer):
@@ -848,7 +869,7 @@ class TaskReadSerializer(serializers.ModelSerializer):
     target_storage = StorageSerializer(required=False, allow_null=True)
     source_storage = StorageSerializer(required=False, allow_null=True)
     jobs = JobsSummarySerializer(url_filter_key='task_id', source='segment_set')
-    labels = LabelsSummarySerializer(url_filter_key='task_id')
+    labels = LabelsSummarySerializer(source='*')
 
     class Meta:
         model = models.Task
@@ -998,6 +1019,10 @@ def update(self, instance, validated_data):
         _update_related_storages(instance, validated_data)
 
         instance.save()
+        instance.task_labels_count = instance.label_set.filter(
+            parent__isnull=True).count()
+        instance.proj_labels_count = instance.project.label_set.filter(
+            parent__isnull=True).count() if instance.project else 0
         return instance
 
     def validate(self, attrs):
@@ -1059,7 +1084,7 @@ class ProjectReadSerializer(serializers.ModelSerializer):
     target_storage = StorageSerializer(required=False, allow_null=True, read_only=True)
     source_storage = StorageSerializer(required=False, allow_null=True, read_only=True)
     tasks = TasksSummarySerializer(models.Task, url_filter_key='project_id')
-    labels = LabelsSummarySerializer(url_filter_key='project_id')
+    labels = LabelsSummarySerializer(source='*')
 
     class Meta:
         model = models.Project
@@ -1137,6 +1162,9 @@ def update(self, instance, validated_data):
         _update_related_storages(instance, validated_data)
 
         instance.save()
+        instance.proj_labels_count = instance.label_set.filter(
+            parent__isnull=True).count()
+
         return instance
 
 class AboutSerializer(serializers.Serializer):

diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py
@@ -21,6 +21,7 @@
 from django.http import HttpResponse, HttpResponseNotFound, HttpResponseBadRequest
 from django.utils import timezone
 import django.db.models as dj_models
+from django.db.models import Count, Q
 
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import (
@@ -229,6 +230,9 @@ class ProjectViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
     ).prefetch_related(
         'tasks', 'label_set__sublabels__attributespec_set',
         'label_set__attributespec_set'
+    ).annotate(
+        proj_labels_count=Count('label',
+            filter=Q(label__parent__isnull=True), distinct=True)
     ).all()
 
     # NOTE: The search_fields attribute should be a list of names of text
@@ -667,7 +671,11 @@ class TaskViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
         completed_jobs_count=dj_models.Count(
             'segment__job',
             filter=dj_models.Q(segment__job__state=models.StateChoice.COMPLETED.value)
-        )
+        ),
+        task_labels_count=Count('label',
+            filter=Q(label__parent__isnull=True), distinct=True),
+        proj_labels_count=Count('project__label',
+            filter=Q(project__label__parent__isnull=True), distinct=True)
     ).all()
 
     lookup_fields = {
@@ -1234,12 +1242,14 @@ def preview(self, request, pk):
 class JobViewSet(viewsets.GenericViewSet, mixins.ListModelMixin,
     mixins.RetrieveModelMixin, PartialUpdateModelMixin, UploadMixin, AnnotationMixin
 ):
-    queryset = Job.objects.select_related('segment__task__data').prefetch_related(
-        'segment__task__label_set', 'segment__task__project__label_set',
-        'segment__task__label_set__sublabels__attributespec_set',
-        'segment__task__project__label_set__sublabels__attributespec_set',
-        'segment__task__label_set__attributespec_set',
-        'segment__task__project__label_set__attributespec_set'
+    queryset = Job.objects.select_related('assignee', 'segment__task__data',
+        'segment__task__project'
+    ).annotate(
+        Count('issues', distinct=True),
+        task_labels_count=Count('segment__task__label',
+            filter=Q(segment__task__label__parent__isnull=True), distinct=True),
+        proj_labels_count=Count('segment__task__project__label',
+            filter=Q(segment__task__project__label__parent__isnull=True), distinct=True)
     ).all()
 
     iam_organization_field = 'segment__task__organization'

diff --git a/cvat/schema.yml b/cvat/schema.yml
@@ -7123,13 +7123,13 @@ components:
     IssuesSummary:
       type: object
       properties:
-        count:
-          type: integer
-          default: 0
         url:
           type: string
           format: uri
           readOnly: true
+        count:
+          type: integer
+          readOnly: true
     JobAnnotationsUpdateRequest:
       oneOf:
       - $ref: '#/components/schemas/LabeledDataRequest'
@@ -7563,13 +7563,13 @@ components:
     LabelsSummary:
       type: object
       properties:
-        count:
-          type: integer
-          default: 0
         url:
           type: string
           format: uri
           readOnly: true
+        count:
+          type: integer
+          readOnly: true
     LocationEnum:
       enum:
       - cloud_storage