From 0d827a81e4cd1998850df6d3777162efe1d30dcb Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 6 Jan 2023 23:34:35 +0200 Subject: [PATCH] Fix pagination in some endpoints --- cvat-sdk/cvat_sdk/core/proxies/issues.py | 5 ++ cvat-sdk/cvat_sdk/core/proxies/jobs.py | 2 +- cvat-sdk/cvat_sdk/core/proxies/projects.py | 3 +- cvat-sdk/cvat_sdk/core/proxies/tasks.py | 4 +- cvat/apps/engine/serializers.py | 8 +-- cvat/apps/engine/utils.py | 43 ++++++++++++++ cvat/apps/engine/views.py | 68 +++++++++++----------- 7 files changed, 92 insertions(+), 41 deletions(-) diff --git a/cvat-sdk/cvat_sdk/core/proxies/issues.py b/cvat-sdk/cvat_sdk/core/proxies/issues.py index 5583fd083c13..3b8c7ec98ccc 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/issues.py +++ b/cvat-sdk/cvat_sdk/core/proxies/issues.py @@ -3,8 +3,10 @@ # SPDX-License-Identifier: MIT from __future__ import annotations +from typing import List from cvat_sdk.api_client import apis, models +from cvat_sdk.core.helpers import get_paginated_collection from cvat_sdk.core.proxies.model_proxy import ( ModelCreateMixin, ModelDeleteMixin, @@ -50,6 +52,9 @@ class Issue( ): _model_partial_update_arg = "patched_issue_write_request" + def get_comments(self) -> List[Comment]: + return [Comment(self._client, m) for m in get_paginated_collection(self.api.list_comments_endpoint, id=self.id)] + class IssuesRepo( _IssueRepoBase, diff --git a/cvat-sdk/cvat_sdk/core/proxies/jobs.py b/cvat-sdk/cvat_sdk/core/proxies/jobs.py index 22235d07a975..3b5f1770d5ef 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/jobs.py +++ b/cvat-sdk/cvat_sdk/core/proxies/jobs.py @@ -161,7 +161,7 @@ def remove_frames_by_ids(self, ids: Sequence[int]) -> None: ) def get_issues(self) -> List[Issue]: - return [Issue(self._client, m) for m in self.api.list_issues(id=self.id)[0]] + return [Issue(self._client, m) for m in get_paginated_collection(self.api.list_issues_endpoint, id=self.id)] def get_commits(self) -> List[models.IJobCommit]: return get_paginated_collection(self.api.list_commits_endpoint, id=self.id) diff --git a/cvat-sdk/cvat_sdk/core/proxies/projects.py b/cvat-sdk/cvat_sdk/core/proxies/projects.py index 0053c15bc443..62311a040fe0 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/projects.py +++ b/cvat-sdk/cvat_sdk/core/proxies/projects.py @@ -11,6 +11,7 @@ from cvat_sdk.api_client import apis, models from cvat_sdk.core.downloading import Downloader +from cvat_sdk.core.helpers import get_paginated_collection from cvat_sdk.core.progress import ProgressReporter from cvat_sdk.core.proxies.model_proxy import ( ModelCreateMixin, @@ -124,7 +125,7 @@ def get_annotations(self) -> models.ILabeledData: return annotations def get_tasks(self) -> List[Task]: - return [Task(self._client, m) for m in self.api.list_tasks(id=self.id)[0].results] + return [Task(self._client, m) for m in get_paginated_collection(self.api.list_tasks_endpoint, id=self.id)] def get_preview( self, diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index 97dcbdbcbdc4..18b5b6215c8e 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -18,6 +18,7 @@ from cvat_sdk.api_client import apis, exceptions, models from cvat_sdk.core import git from cvat_sdk.core.downloading import Downloader +from cvat_sdk.core.helpers import get_paginated_collection from cvat_sdk.core.progress import ProgressReporter from cvat_sdk.core.proxies.annotations import AnnotationCrudMixin from cvat_sdk.core.proxies.jobs import Job @@ -301,7 +302,8 @@ def download_backup( self._client.logger.info(f"Backup for task {self.id} has been downloaded to {filename}") def get_jobs(self) -> List[Job]: - return [Job(self._client, m) for m in self.api.list_jobs(id=self.id)[0]] + return [Job(self._client, model=m) + for m in get_paginated_collection(self.api.list_jobs_endpoint, id=self.id)] def get_meta(self) -> models.IDataMetaRead: (meta, _) = self.api.retrieve_data_meta(self.id) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index abb29bd36e3e..da0139dba53c 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -184,14 +184,14 @@ class JobReadSerializer(serializers.ModelSerializer): project_id = serializers.ReadOnlyField(source="get_project_id", allow_null=True) start_frame = serializers.ReadOnlyField(source="segment.start_frame") stop_frame = serializers.ReadOnlyField(source="segment.stop_frame") - assignee = BasicUserSerializer(allow_null=True) - dimension = serializers.CharField(max_length=2, source='segment.task.dimension') - labels = LabelSerializer(many=True, source='get_labels') + assignee = BasicUserSerializer(allow_null=True, read_only=True) + dimension = serializers.CharField(max_length=2, source='segment.task.dimension', read_only=True) + labels = LabelSerializer(many=True, source='get_labels', read_only=True) data_chunk_size = serializers.ReadOnlyField(source='segment.task.data.chunk_size') data_compressed_chunk_type = serializers.ReadOnlyField(source='segment.task.data.compressed_chunk_type') mode = serializers.ReadOnlyField(source='segment.task.mode') bug_tracker = serializers.CharField(max_length=2000, source='get_bug_tracker', - allow_null=True) + allow_null=True, read_only=True) class Meta: model = models.Job diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index c2dd82ce7d2e..b9879b028226 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT import ast +# from typing import Optional, Type import cv2 as cv from collections import namedtuple import hashlib @@ -17,6 +18,11 @@ from PIL import Image from django.core.exceptions import ValidationError +from django.urls import reverse as _django_reverse +from django.utils.http import urlencode +# from django.db.models.query import QuerySet +# from rest_framework.response import Response +# from rest_framework.viewsets import GenericViewSet Import = namedtuple("Import", ["module", "name", "alias"]) @@ -146,3 +152,40 @@ def configure_dependent_job(queue, rq_id, rq_func, db_storage, filename, key): job_id=rq_job_id_download_file ) return rq_job_download_file + +# def make_paginated_response(queryset: QuerySet, *, +# viewset: GenericViewSet, +# response_type: Type[Response] = Response, +# request: Optional[Request] = None, +# **serializer_params +# ) -> Response: +# # Adapted from the mixins.ListModelMixin.list() + +# serializer_params.setdefault('many', True) + +# if request is not None: +# context = serializer_params.setdefault('context', {}) +# context.setdefault('request', request) + +# make_serializer = viewset.get_serializer + +# page = viewset.paginate_queryset(queryset) +# if page is not None: +# serializer = make_serializer(page, **serializer_params) +# return viewset.get_paginated_response(serializer.data) + +# serializer = make_serializer(queryset, **serializer_params) + +# return response_type(serializer.data) + +def reverse(viewname, *, args=None, kwargs=None, query_params=None) -> str: + """ + The same as reverse(), but adds query params support. + """ + + url = _django_reverse(viewname, args=args, kwargs=kwargs) + + if query_params: + return f'{url}?{urlencode(query_params)}' + + return url diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index aa3ecd1438b2..8a1c4946da40 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -64,7 +64,7 @@ from utils.dataset_manifest import ImageManifestManager from cvat.apps.engine.utils import ( - av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message + av_scan_paths, process_failed_job, configure_dependent_job, parse_exception_message, reverse ) from cvat.apps.engine import backup from cvat.apps.engine.mixins import PartialUpdateModelMixin, UploadMixin, AnnotationMixin, SerializeMixin, DestroyModelMixin, CreateModelMixin @@ -898,15 +898,19 @@ def perform_destroy(self, instance): @extend_schema(summary='Method returns a list of jobs for a specific task', - responses=JobReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer(many=True), - # Remove regular list() parameters from swagger schema - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + responses=JobReadSerializer(many=True)) + @action(detail=True, methods=['GET'], serializer_class=JobReadSerializer) def jobs(self, request, pk): self.get_object() # force to call check_object_permissions - queryset = Job.objects.filter(segment__task_id=pk) - serializer = JobReadSerializer(queryset, many=True, + queryset = Job.objects.filter(segment__task_id=pk).order_by('id') + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True, + context={"request": request}) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True, context={"request": request}) return Response(serializer.data) @@ -1650,20 +1654,14 @@ def dataset_export(self, request, pk): callback=dm.views.export_job_as_dataset ) - @extend_schema(summary='Method returns list of issues for the job', - responses=IssueReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer(many=True), - # Remove regular list() parameters from swagger schema - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + @extend_schema(summary='Moved to GET api/issues', + deprecated=True) # TODO: to be removed in v2.5 + @action(detail=True, methods=['GET'], serializer_class=IssueReadSerializer) def issues(self, request, pk): - db_job = self.get_object() - queryset = db_job.issues - serializer = IssueReadSerializer(queryset, - context={'request': request}, many=True) - - return Response(serializer.data) - + # https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other + return Response(status=status.HTTP_303_SEE_OTHER, headers={ + 'Location': reverse('issue-list', query_params={'job_id': pk}) + }) @extend_schema(summary='Method returns data for a specific job', parameters=[ @@ -1705,7 +1703,7 @@ def data(self, request, pk): @action(detail=True, methods=['GET', 'PATCH'], serializer_class=DataMetaReadSerializer, url_path='data/meta') def metadata(self, request, pk): - self.get_object() #force to call check_object_permissions + self.get_object() # force to call check_object_permissions db_job = models.Job.objects.prefetch_related( 'segment', 'segment__task', @@ -1768,17 +1766,17 @@ def metadata(self, request, pk): responses={ '200': JobCommitSerializer(many=True), }) - @action(detail=True, methods=['GET'], serializer_class=None) + @action(detail=True, methods=['GET'], serializer_class=JobCommitSerializer) def commits(self, request, pk): db_job = self.get_object() queryset = db_job.commits.order_by('-id') page = self.paginate_queryset(queryset) if page is not None: - serializer = JobCommitSerializer(page, context={'request': request}, many=True) + serializer = self.get_serializer(page, context={'request': request}, many=True) return self.get_paginated_response(serializer.data) - serializer = JobCommitSerializer(queryset, context={'request': request}, many=True) + serializer = self.get_serializer(queryset, context={'request': request}, many=True) return Response(serializer.data) @extend_schema(summary='Method returns a preview image for the job', @@ -1864,17 +1862,19 @@ def perform_create(self, serializer, **kwargs): super().perform_create(serializer, owner=self.request.user) @extend_schema(summary='The action returns all comments of a specific issue', - responses=CommentReadSerializer(many=True)) # Duplicate to still get 'list' op. name - @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer(many=True), - # Remove regular list() parameters from swagger schema - # https://drf-spectacular.readthedocs.io/en/latest/faq.html#my-action-is-erroneously-paginated-or-has-filter-parameters-that-i-do-not-want - pagination_class=None, filter_fields=None, search_fields=None, ordering_fields=None) + responses=CommentReadSerializer(many=True)) + @action(detail=True, methods=['GET'], serializer_class=CommentReadSerializer) def comments(self, request, pk): - # TODO: remove this endpoint? It is totally covered by issue body. - db_issue = self.get_object() - queryset = db_issue.comments - serializer = CommentReadSerializer(queryset, + queryset = db_issue.comments.order_by('-id') + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True, + context={"request": request}) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, context={'request': request}, many=True) return Response(serializer.data)