refactor: downstream entity links api [FC-0076]

cms/djangoapps/contentstore/models.py

@@ -7,8 +7,10 @@
 
 from config_models.models import ConfigurationModel
 from django.db import models
-from django.db.models import QuerySet
+from django.db.models import Count, F, Q, QuerySet
 from django.db.models.fields import IntegerField, TextField
+from django.db.models.functions import Coalesce
+from django.db.models.lookups import GreaterThan
 from django.utils.translation import gettext_lazy as _
 from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField
 from opaque_keys.edx.keys import CourseKey, UsageKey
@@ -173,7 +175,12 @@ def update_or_create(
             )
         try:
             link = cls.objects.get(downstream_usage_key=downstream_usage_key)
-            has_changes = False
+            # TODO: until we save modified datetime for course xblocks in index, the modified time for links are updated
+            # everytime a downstream/course block is updated. This allows us to order links[1] based on recently
+            # modified downstream version.
+            # pylint: disable=line-too-long
+            # 1. https://github.com/open-craft/frontend-app-course-authoring/blob/0443d88824095f6f65a3a64b77244af590d4edff/src/course-libraries/ReviewTabContent.tsx#L222-L233
+            has_changes = True  # change to false once above condition is met.
             for key, value in new_values.items():
                 prev = getattr(link, key)
                 # None != None is True, so we need to check for it specially
@@ -191,16 +198,31 @@ def update_or_create(
         return link
 
     @classmethod
-    def get_by_downstream_context(cls, downstream_context_key: CourseKey) -> QuerySet["PublishableEntityLink"]:
+    def filter_links(
+        cls,
+        **link_filter,
+    ) -> QuerySet["PublishableEntityLink"]:
         """
-        Get all links for given downstream context, preselects related published version and learning package.
+        Get all links along with sync flag, upstream context title and version, with optional filtering.
         """
-        return cls.objects.filter(
-            downstream_context_key=downstream_context_key
-        ).select_related(
+        ready_to_sync = link_filter.pop('ready_to_sync', None)
+        result = cls.objects.filter(**link_filter).select_related(
             "upstream_block__published__version",
             "upstream_block__learning_package"
+        ).annotate(
+            ready_to_sync=(
+                GreaterThan(
+                    Coalesce("upstream_block__published__version__version_num", 0),
+                    Coalesce("version_synced", 0)
+                ) & GreaterThan(
+                    Coalesce("upstream_block__published__version__version_num", 0),
+                    Coalesce("version_declined", 0)
+                )
+            )
         )
+        if ready_to_sync is not None:
+            result = result.filter(ready_to_sync=ready_to_sync)
+        return result
 
     @classmethod
     def get_by_upstream_usage_key(cls, upstream_usage_key: UsageKey) -> QuerySet["PublishableEntityLink"]:
@@ -211,6 +233,35 @@ def get_by_upstream_usage_key(cls, upstream_usage_key: UsageKey) -> QuerySet["Pu
             upstream_usage_key=upstream_usage_key,
         )
 
+    @classmethod
+    def summarize_by_downstream_context(cls, downstream_context_key: CourseKey) -> QuerySet:
+        """
+        Returns a summary of links by upstream context for given downstream_context_key.
+        Example:
+        [
+            {
+                "upstream_context_title": "CS problems 3",
+                "upstream_context_key": "lib:OpenedX:CSPROB3",
+                "ready_to_sync_count": 11,
+                "total_count": 14
+            },
+            {
+                "upstream_context_title": "CS problems 2",
+                "upstream_context_key": "lib:OpenedX:CSPROB2",
+                "ready_to_sync_count": 15,
+                "total_count": 24
+            },
+        ]
+        """
+        result = cls.filter_links(downstream_context_key=downstream_context_key).values(
+            "upstream_context_key",
+            upstream_context_title=F("upstream_block__learning_package__title"),
+        ).annotate(
+            ready_to_sync_count=Count("id", Q(ready_to_sync=True)),
+            total_count=Count('id')
+        )
+        return result
+
 
 class LearningContextLinksStatusChoices(models.TextChoices):
     """

cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py

@@ -2,6 +2,12 @@
 
 from cms.djangoapps.contentstore.rest_api.v2.serializers.downstreams import (
     PublishableEntityLinksSerializer,
-    PublishableEntityLinksUsageKeySerializer,
+    PublishableEntityLinksSummarySerializer,
 )
 from cms.djangoapps.contentstore.rest_api.v2.serializers.home import CourseHomeTabSerializerV2
+
+__all__ = [
+    'CourseHomeTabSerializerV2',
+    'PublishableEntityLinksSerializer',
+    'PublishableEntityLinksSummarySerializer',
+]

cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py

@@ -13,28 +13,18 @@ class PublishableEntityLinksSerializer(serializers.ModelSerializer):
     """
     upstream_context_title = serializers.CharField(read_only=True)
     upstream_version = serializers.IntegerField(read_only=True)
-    ready_to_sync = serializers.SerializerMethodField()
-
-    def get_ready_to_sync(self, obj):
-        """Calculate ready_to_sync field"""
-        return bool(
-            obj.upstream_version and
-            obj.upstream_version > (obj.version_synced or 0) and
-            obj.upstream_version > (obj.version_declined or 0)
-        )
+    ready_to_sync = serializers.BooleanField()
 
     class Meta:
         model = PublishableEntityLink
         exclude = ['upstream_block', 'uuid']
 
 
-class PublishableEntityLinksUsageKeySerializer(serializers.ModelSerializer):
+class PublishableEntityLinksSummarySerializer(serializers.Serializer):
     """
-    Serializer for returning a string list of the usage keys.
+    Serializer for summary for publishable entity links
     """
-    def to_representation(self, instance: PublishableEntityLink) -> str:
-        return str(instance.downstream_usage_key)
-
-    class Meta:
-        model = PublishableEntityLink
-        fields = ('downstream_usage_key')
+    upstream_context_title = serializers.CharField(read_only=True)
+    upstream_context_key = serializers.CharField(read_only=True)
+    ready_to_sync_count = serializers.IntegerField(read_only=True)
+    total_count = serializers.IntegerField(read_only=True)

cms/djangoapps/contentstore/rest_api/v2/urls.py

@@ -13,26 +13,20 @@
         home.HomePageCoursesViewV2.as_view(),
         name="courses",
     ),
-    # TODO: Potential future path.
-    # re_path(
-    #     fr'^downstreams/$',
-    #     downstreams.DownstreamsListView.as_view(),
-    #     name="downstreams_list",
-    # ),
+    re_path(
+        r'^downstreams/$',
+        downstreams.DownstreamListView.as_view(),
+        name="downstreams_list",
+    ),
     re_path(
         fr'^downstreams/{settings.USAGE_KEY_PATTERN}$',
         downstreams.DownstreamView.as_view(),
         name="downstream"
     ),
     re_path(
-        f'^upstreams/{settings.COURSE_KEY_PATTERN}$',
-        downstreams.UpstreamListView.as_view(),
-        name='upstream-list'
-    ),
-    re_path(
-        f'^upstream/{settings.USAGE_KEY_PATTERN}/downstream-links$',
-        downstreams.DownstreamContextListView.as_view(),
-        name='downstream-link-list'
+        f'^downstreams/{settings.COURSE_KEY_PATTERN}/summary$',
+        downstreams.DownstreamSummaryView.as_view(),
+        name='upstream-summary-list'
     ),
     re_path(
         fr'^downstreams/{settings.USAGE_KEY_PATTERN}/sync$',

cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py

@@ -45,11 +45,29 @@
     GET: List all downstream blocks linked to a library block.
         200: A list of downstream usage_keys linked to the library block.
 
-  # NOT YET IMPLEMENTED -- Will be needed for full Libraries Relaunch in ~Teak.
   /api/contentstore/v2/downstreams
   /api/contentstore/v2/downstreams?course_id=course-v1:A+B+C&ready_to_sync=true
       GET: List downstream blocks that can be synced, filterable by course or sync-readiness.
-        200: A paginated list of applicable & accessible downstream blocks. Entries are UpstreamLinks.
+        200: A paginated list of applicable & accessible downstream blocks. Entries are PublishableEntityLinks.
+
+  /api/contentstore/v2/downstreams/<course_key>/summary
+      GET: List summary of links by course key
+        200: A list of summary of links by course key
+        Example:
+        [
+            {
+                "upstream_context_title": "CS problems 3",
+                "upstream_context_key": "lib:OpenedX:CSPROB3",
+                "ready_to_sync_count": 11,
+                "total_count": 14
+            },
+            {
+                "upstream_context_title": "CS problems 2",
+                "upstream_context_key": "lib:OpenedX:CSPROB2",
+                "ready_to_sync_count": 15,
+                "total_count": 24
+            },
+        ]
 
 UpstreamLink response schema:
   {
@@ -66,9 +84,11 @@
 
 from attrs import asdict as attrs_asdict
 from django.contrib.auth.models import User  # pylint: disable=imported-auth-user
+from edx_rest_framework_extensions.paginators import DefaultPagination
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey, UsageKey
 from rest_framework.exceptions import NotFound, ValidationError
+from rest_framework.fields import BooleanField
 from rest_framework.request import Request
 from rest_framework.response import Response
 from rest_framework.views import APIView
@@ -78,7 +98,7 @@
 from cms.djangoapps.contentstore.models import PublishableEntityLink
 from cms.djangoapps.contentstore.rest_api.v2.serializers import (
     PublishableEntityLinksSerializer,
-    PublishableEntityLinksUsageKeySerializer,
+    PublishableEntityLinksSummarySerializer,
 )
 from cms.lib.xblock.upstream_sync import (
     BadDownstream,
@@ -96,9 +116,9 @@
     DeveloperErrorViewMixin,
     view_auth_classes,
 )
-from xmodule.video_block.transcripts_utils import clear_transcripts
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.video_block.transcripts_utils import clear_transcripts
 
 logger = logging.getLogger(__name__)
 
@@ -113,57 +133,92 @@ class _AuthenticatedRequest(Request):
     user: User
 
 
-# TODO: Potential future view.
-# @view_auth_classes(is_authenticated=True)
-# class DownstreamListView(DeveloperErrorViewMixin, APIView):
-#     """
-#     List all blocks which are linked to upstream content, with optional filtering.
-#     """
-#     def get(self, request: _AuthenticatedRequest) -> Response:
-#         """
-#         Handle the request.
-#         """
-#         course_key_string = request.GET['course_id']
-#         syncable = request.GET['ready_to_sync']
-#         ...
+class DownstreamListPaginator(DefaultPagination):
+    """Custom paginator for downstream entity links"""
+    page_size = 100
+    max_page_size = 1000
+
+    def paginate_queryset(self, queryset, request, view=None):
+        if 'no_page' in request.query_params:
+            return queryset
+
+        return super().paginate_queryset(queryset, request, view)
+
+    def get_paginated_response(self, data, *args, **kwargs):
+        if 'no_page' in args[0].query_params:
+            return Response(data)
+        response = super().get_paginated_response(data)
+        # replace next and previous links by next and previous page number
+        response.data.update({
+            'next_page_num': self.page.next_page_number() if self.page.has_next() else None,
+            'previous_page_num': self.page.previous_page_number() if self.page.has_previous() else None,
+        })
+        return response
 
 
 @view_auth_classes()
-class UpstreamListView(DeveloperErrorViewMixin, APIView):
+class DownstreamListView(DeveloperErrorViewMixin, APIView):
     """
-    Serves course->library publishable entity links
+    List all blocks which are linked to an upstream context, with optional filtering.
     """
-    def get(self, request: _AuthenticatedRequest, course_key_string: str):
+
+    def get(self, request: _AuthenticatedRequest):
         """
         Fetches publishable entity links for given course key
         """
-        try:
-            course_key = CourseKey.from_string(course_key_string)
-        except InvalidKeyError as exc:
-            raise ValidationError(detail=f"Malformed course key: {course_key_string}") from exc
-        links = PublishableEntityLink.get_by_downstream_context(downstream_context_key=course_key)
-        serializer = PublishableEntityLinksSerializer(links, many=True)
-        return Response(serializer.data)
+        course_key_string = request.GET.get('course_id')
+        ready_to_sync = request.GET.get('ready_to_sync')
+        upstream_usage_key = request.GET.get('upstream_usage_key')
+        link_filter: dict[str, CourseKey | UsageKey | bool] = {}
+        paginator = DownstreamListPaginator()
+        if course_key_string:
+            try:
+                link_filter["downstream_context_key"] = CourseKey.from_string(course_key_string)
+            except InvalidKeyError as exc:
+                raise ValidationError(detail=f"Malformed course key: {course_key_string}") from exc
+        if ready_to_sync is not None:
+            link_filter["ready_to_sync"] = BooleanField().to_internal_value(ready_to_sync)
+        if upstream_usage_key:
+            try:
+                link_filter["upstream_usage_key"] = UsageKey.from_string(upstream_usage_key)
+            except InvalidKeyError as exc:
+                raise ValidationError(detail=f"Malformed usage key: {upstream_usage_key}") from exc
+        links = PublishableEntityLink.filter_links(**link_filter)
+        paginated_links = paginator.paginate_queryset(links, self.request, view=self)
+        serializer = PublishableEntityLinksSerializer(paginated_links, many=True)
+        return paginator.get_paginated_response(serializer.data, self.request)
 
 
 @view_auth_classes()
-class DownstreamContextListView(DeveloperErrorViewMixin, APIView):
+class DownstreamSummaryView(DeveloperErrorViewMixin, APIView):
     """
-    Serves library block->downstream usage keys
+    Serves course->library publishable entity links summary
     """
-    def get(self, request: _AuthenticatedRequest, usage_key_string: str) -> Response:
+    def get(self, request: _AuthenticatedRequest, course_key_string: str):
         """
-        Fetches downstream links for given publishable entity
+        Fetches publishable entity links summary for given course key
+        Example:
+        [
+            {
+                "upstream_context_title": "CS problems 3",
+                "upstream_context_key": "lib:OpenedX:CSPROB3",
+                "ready_to_sync_count": 11,
+                "total_count": 14
+            },
+            {
+                "upstream_context_title": "CS problems 2",
+                "upstream_context_key": "lib:OpenedX:CSPROB2",
+                "ready_to_sync_count": 15,
+                "total_count": 24
+            },
+        ]
         """
         try:
-            usage_key = UsageKey.from_string(usage_key_string)
+            course_key = CourseKey.from_string(course_key_string)
         except InvalidKeyError as exc:
-            raise ValidationError(detail=f"Malformed usage key: {usage_key_string}") from exc
-
-        links = PublishableEntityLink.get_by_upstream_usage_key(upstream_usage_key=usage_key)
-
-        serializer = PublishableEntityLinksUsageKeySerializer(links, many=True)
-
+            raise ValidationError(detail=f"Malformed course key: {course_key_string}") from exc
+        links = PublishableEntityLink.summarize_by_downstream_context(downstream_context_key=course_key)
+        serializer = PublishableEntityLinksSummarySerializer(links, many=True)
         return Response(serializer.data)
 
 
@@ -231,7 +286,7 @@ def delete(self, request: _AuthenticatedRequest, usage_key_string: str) -> Respo
         downstream = _load_accessible_block(request.user, usage_key_string, require_write_access=True)
         try:
             sever_upstream_link(downstream)
-        except NoUpstream as exc:
+        except NoUpstream:
             logger.exception(
                 "Tried to DELETE upstream link of '%s', but it wasn't linked to anything in the first place. "
                 "Will do nothing. ",