refactor: Refactor upstream links summary to add the top-level parent logic [FC-0097]

cms/djangoapps/contentstore/models.py

@@ -7,8 +7,8 @@
 
 from config_models.models import ConfigurationModel
 from django.db import models
-from django.db.models import Count, F, Q, QuerySet, Max
-from django.db.models.fields import IntegerField, TextField
+from django.db.models import QuerySet, OuterRef, Case, When, Exists, Value, ExpressionWrapper
+from django.db.models.fields import IntegerField, TextField, BooleanField
 from django.db.models.functions import Coalesce
 from django.db.models.lookups import GreaterThan
 from django.utils.translation import gettext_lazy as _
@@ -111,6 +111,20 @@ class EntityLinkBase(models.Model):
     created = manual_date_time_field()
     updated = manual_date_time_field()
 
+    @property
+    def upstream_context_title(self) -> str:
+        """
+        Returns upstream context title.
+        """
+        raise NotImplementedError
+
+    @property
+    def published_at(self) -> str | None:
+        """
+        Returns the published date of the entity
+        """
+        raise NotImplementedError
+
     class Meta:
         abstract = True
 
@@ -157,6 +171,15 @@ def upstream_context_title(self) -> str:
         """
         return self.upstream_block.publishable_entity.learning_package.title
 
+    @property
+    def published_at(self) -> str | None:
+        """
+        Returns the published date of the component
+        """
+        if self.upstream_block.publishable_entity.published is None:
+            raise AttributeError(_("The component must be published to access `published_at`"))
+        return self.upstream_block.publishable_entity.published.publish_log_record.publish_log.published_at
+
     @classmethod
     def filter_links(
         cls,
@@ -189,7 +212,9 @@ def filter_links(
                     Coalesce("upstream_block__publishable_entity__published__version__version_num", 0),
                     Coalesce("version_declined", 0)
                 )
-            )
+            ),
+            # This is alwys False, the components doens't have children
+            ready_to_sync_from_children=Value(False, output_field=BooleanField())
         )
         if ready_to_sync is not None:
             result = result.filter(ready_to_sync=ready_to_sync)
@@ -216,40 +241,6 @@ def filter_links(
 
         return result
 
-    @classmethod
-    def summarize_by_downstream_context(cls, downstream_context_key: CourseKey) -> QuerySet:
-        """
-        Returns a summary of links by upstream context for given downstream_context_key.
-        Example:
-        [
-            {
-                "upstream_context_title": "CS problems 3",
-                "upstream_context_key": "lib:OpenedX:CSPROB3",
-                "ready_to_sync_count": 11,
-                "total_count": 14,
-                "last_published_at": "2025-05-02T20:20:44.989042Z"
-            },
-            {
-                "upstream_context_title": "CS problems 2",
-                "upstream_context_key": "lib:OpenedX:CSPROB2",
-                "ready_to_sync_count": 15,
-                "total_count": 24,
-                "last_published_at": "2025-05-03T21:20:44.989042Z"
-            },
-        ]
-        """
-        result = cls.filter_links(downstream_context_key=downstream_context_key).values(
-            "upstream_context_key",
-            upstream_context_title=F("upstream_block__publishable_entity__learning_package__title"),
-        ).annotate(
-            ready_to_sync_count=Count("id", Q(ready_to_sync=True)),
-            total_count=Count("id"),
-            last_published_at=Max(
-                "upstream_block__publishable_entity__published__publish_log_record__publish_log__published_at"
-            )
-        )
-        return result
-
     @classmethod
     def update_or_create(
         cls,
@@ -351,6 +342,15 @@ def upstream_context_title(self) -> str:
         """
         return self.upstream_container.publishable_entity.learning_package.title
 
+    @property
+    def published_at(self) -> str | None:
+        """
+        Returns the published date of the container
+        """
+        if self.upstream_container.publishable_entity.published is None:
+            raise AttributeError(_("The container must be published to access `published_at`"))
+        return self.upstream_container.publishable_entity.published.publish_log_record.publish_log.published_at
+
     @classmethod
     def filter_links(
         cls,
@@ -402,6 +402,54 @@ def filter_links(
 
     @classmethod
     def _annotate_query_with_ready_to_sync(cls, query_set: QuerySet["EntityLinkBase"]) -> QuerySet["EntityLinkBase"]:
+        """
+        Adds ready to sync related values to the query set:
+        * `ready_to_sync`: When the container is ready to sync.
+        * `ready_to_sync_from_children`: When any children is ready to sync.
+        """
+        # SubQuery to verify if some container children (associated with top-level parent)
+        # needs sync.
+        subq_container = cls.objects.filter(
+            top_level_parent=OuterRef('pk')
+        ).annotate(
+            child_ready=Case(
+                When(
+                    GreaterThan(
+                        Coalesce("upstream_container__publishable_entity__published__version__version_num", 0),
+                        Coalesce("version_synced", 0)
+                    ) & GreaterThan(
+                        Coalesce("upstream_container__publishable_entity__published__version__version_num", 0),
+                        Coalesce("version_declined", 0)
+                    ),
+                    then=1
+                ),
+                default=0,
+                output_field=models.IntegerField()
+            )
+        ).filter(child_ready=1)
+
+        # SubQuery to verify if some component children (assisiated with top-level parent)
+        # needs sync.
+        subq_components = ComponentLink.objects.filter(
+            top_level_parent=OuterRef('pk')
+        ).annotate(
+            child_ready=Case(
+                When(
+                    GreaterThan(
+                        Coalesce("upstream_block__publishable_entity__published__version__version_num", 0),
+                        Coalesce("version_synced", 0)
+                    ) & GreaterThan(
+                        Coalesce("upstream_block__publishable_entity__published__version__version_num", 0),
+                        Coalesce("version_declined", 0)
+                    ),
+                    then=1
+                ),
+                default=0,
+                output_field=models.IntegerField()
+            )
+        ).filter(child_ready=1)
+
+        # TODO: is there a way to run `subq_container` or `subq_components` depending on the container type?
         return query_set.annotate(
             ready_to_sync=(
                 GreaterThan(
@@ -411,42 +459,12 @@ def _annotate_query_with_ready_to_sync(cls, query_set: QuerySet["EntityLinkBase"
                     Coalesce("upstream_container__publishable_entity__published__version__version_num", 0),
                     Coalesce("version_declined", 0)
                 )
-            )
-        )
-
-    @classmethod
-    def summarize_by_downstream_context(cls, downstream_context_key: CourseKey) -> QuerySet:
-        """
-        Returns a summary of links by upstream context for given downstream_context_key.
-        Example:
-        [
-            {
-                "upstream_context_title": "CS problems 3",
-                "upstream_context_key": "lib:OpenedX:CSPROB3",
-                "ready_to_sync_count": 11,
-                "total_count": 14,
-                "last_published_at": "2025-05-02T20:20:44.989042Z"
-            },
-            {
-                "upstream_context_title": "CS problems 2",
-                "upstream_context_key": "lib:OpenedX:CSPROB2",
-                "ready_to_sync_count": 15,
-                "total_count": 24,
-                "last_published_at": "2025-05-03T21:20:44.989042Z"
-            },
-        ]
-        """
-        result = cls.filter_links(downstream_context_key=downstream_context_key).values(
-            "upstream_context_key",
-            upstream_context_title=F("upstream_container__publishable_entity__learning_package__title"),
-        ).annotate(
-            ready_to_sync_count=Count("id", Q(ready_to_sync=True)),
-            total_count=Count('id'),
-            last_published_at=Max(
-                "upstream_container__publishable_entity__published__publish_log_record__publish_log__published_at"
-            )
+            ),
+            ready_to_sync_from_children=ExpressionWrapper(
+                Exists(subq_container) | Exists(subq_components),
+                output_field=BooleanField(),
+            ),
         )
-        return result
 
     @classmethod
     def update_or_create(

cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py

@@ -14,6 +14,7 @@ class ComponentLinksSerializer(serializers.ModelSerializer):
     upstream_context_title = serializers.CharField(read_only=True)
     upstream_version = serializers.IntegerField(read_only=True, source="upstream_version_num")
     ready_to_sync = serializers.BooleanField()
+    ready_to_sync_from_children = serializers.BooleanField()
     top_level_parent_usage_key = serializers.CharField(
         source='top_level_parent.downstream_usage_key',
         read_only=True,
@@ -43,6 +44,7 @@ class ContainerLinksSerializer(serializers.ModelSerializer):
     upstream_context_title = serializers.CharField(read_only=True)
     upstream_version = serializers.IntegerField(read_only=True, source="upstream_version_num")
     ready_to_sync = serializers.BooleanField()
+    ready_to_sync_from_children = serializers.BooleanField()
     top_level_parent_usage_key = serializers.CharField(
         source='top_level_parent.downstream_usage_key',
         read_only=True,

cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py

@@ -239,6 +239,8 @@ def get(self, request: _AuthenticatedRequest):
                     raise ValidationError(detail=f"Malformed key: {upstream_key}") from exc
         links: list[EntityLinkBase] | QuerySet[EntityLinkBase] = []
         if item_type is None or item_type == 'all':
+            # itertools.chain() efficiently concatenates multiple iterables into one iterator,
+            # yielding items from each in sequence without creating intermediate lists.
             links = list(chain(
                 ComponentLink.filter_links(**link_filter),
                 ContainerLink.filter_links(**link_filter)
@@ -346,34 +348,84 @@ def get(self, request: _AuthenticatedRequest, course_key_string: str):
             course_key = CourseKey.from_string(course_key_string)
         except InvalidKeyError as exc:
             raise ValidationError(detail=f"Malformed course key: {course_key_string}") from exc
-        component_links = ComponentLink.summarize_by_downstream_context(downstream_context_key=course_key)
-        container_links = ContainerLink.summarize_by_downstream_context(downstream_context_key=course_key)
-
-        merged = {}
-
-        def process_list(lst):
-            """
-            Process a list to merge it with values in `merged`
-            """
-            for item in lst:
-                key = item["upstream_context_key"]
-                if key not in merged:
-                    merged[key] = item.copy()
-                else:
-                    merged[key]["ready_to_sync_count"] += item["ready_to_sync_count"]
-                    merged[key]["total_count"] += item["total_count"]
-                    if item["last_published_at"] > merged[key]["last_published_at"]:
-                        merged[key]["last_published_at"] = item["last_published_at"]
-
-        # Merge `component_links` and `container_links` by adding the values of
-        # `ready_to_sync_count` and `total_count` of each library.
-        process_list(component_links)
-        process_list(container_links)
-
-        links = list(merged.values())
-        serializer = PublishableEntityLinksSummarySerializer(links, many=True)
+
+        if not has_studio_read_access(request.user, course_key):
+            raise PermissionDenied
+
+        # Gets all links of the Course, using the
+        # top-level parents filter (see `filter_links()` for more info about top-level parents).
+        # `itertools.chain()` efficiently concatenates multiple iterables into one iterator,
+        # yielding items from each in sequence without creating intermediate lists.
+        links = list(chain(
+            ComponentLink.filter_links(
+                downstream_context_key=course_key,
+                use_top_level_parents=True,
+            ),
+            ContainerLink.filter_links(
+                downstream_context_key=course_key,
+                use_top_level_parents=True,
+            ),
+        ))
+
+        # Delete duplicates. From `ComponentLink` and `ContainerLink`
+        # repeated containers may come in this case:
+        # If we have a `Unit A` and a `Component B`, if you update and publish
+        # both, form `ComponentLink` and `ContainerLink` you get the same `Unit A`.
+        links = self._remove_duplicates(links)
+        result = {}
+
+        for link in links:
+            # We iterate each list to do the counting by Library (`context_key`)
+            context_key = link.upstream_context_key
+
+            if context_key not in result:
+                result[context_key] = {
+                    "upstream_context_key": context_key,
+                    "upstream_context_title": link.upstream_context_title,
+                    "ready_to_sync_count": 0,
+                    "total_count": 0,
+                    "last_published_at": None,
+                }
+
+            # Total count
+            result[context_key]["total_count"] += 1
+
+            # Ready to sync count, it also checks if the container has
+            # descendants that need sync (`ready_to_sync_from_children`).
+            if link.ready_to_sync or link.ready_to_sync_from_children:  # type: ignore[attr-defined]
+                result[context_key]["ready_to_sync_count"] += 1
+
+            # The Max `published_at` value
+            # An AttributeError may be thrown if copied/pasted an unpublished item from library to course.
+            # That case breaks all the course library sync page.
+            # TODO: Delete this `try` after avoid copy/paster unpublished items.
+            try:
+                published_at = link.published_at
+            except AttributeError:
+                published_at = None
+            if published_at is not None and (
+                result[context_key]["last_published_at"] is None
+                or result[context_key]["last_published_at"] < published_at
+            ):
+                result[context_key]["last_published_at"] = published_at
+
+        serializer = PublishableEntityLinksSummarySerializer(list(result.values()), many=True)
         return Response(serializer.data)
 
+    def _remove_duplicates(self, links: list[EntityLinkBase]) -> list[EntityLinkBase]:
+        """
+        Remove duplicates based on `EntityLinkBase.downstream_usage_key`
+        """
+        seen_keys = set()
+        unique_links = []
+
+        for link in links:
+            if link.downstream_usage_key not in seen_keys:
+                seen_keys.add(link.downstream_usage_key)
+                unique_links.append(link)
+
+        return unique_links
+
 
 @view_auth_classes(is_authenticated=True)
 class DownstreamView(DeveloperErrorViewMixin, APIView):