Skip to content

feat: Enhance course optimizer to detect previous run links and expand scanning scope #37104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Faraz32123 merged 5 commits into from
Aug 8, 2025
Merged

feat: Enhance course optimizer to detect previous run links and expand scanning scope #37104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5d26e21
feat: Enhance course optimizer to detect previous run links and expan…
djoseph-apphelix Aug 1, 2025
6d0bb37
fix: apply linting rules and refactor code
djoseph-apphelix Aug 5, 2025
457b204
fix: refactor code based on pep-8 guideline
djoseph-apphelix Aug 7, 2025
9449280
fix: added exception handling for links descriptor generator function
djoseph-apphelix Aug 8, 2025
b8bd541
fix: refactor API response to handle empty data
djoseph-apphelix Aug 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
343 changes: 316 additions & 27 deletions cms/djangoapps/contentstore/core/course_optimizer_provider.py
View file
Open in desktop

Large diffs are not rendered by default.

156 changes: 153 additions & 3 deletions cms/djangoapps/contentstore/core/tests/test_course_optimizer_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
"""
Tests for course optimizer
"""

from unittest import mock
from unittest.mock import Mock

from cms.djangoapps.contentstore.tests.utils import CourseTestCase
from opaque_keys.edx.keys import CourseKey

from cms.djangoapps.contentstore.core.course_optimizer_provider import (
_update_node_tree_and_dictionary,
_create_dto_recursive,
_update_node_tree_and_dictionary,
generate_broken_links_descriptor,
sort_course_sections
)
from cms.djangoapps.contentstore.tasks import LinkState
from cms.djangoapps.contentstore.tasks import LinkState, extract_content_URLs_from_course
from cms.djangoapps.contentstore.tests.utils import CourseTestCase
from cms.djangoapps.contentstore.utils import contains_previous_course_reference
from xmodule.tabs import StaticTab


class TestLinkCheckProvider(CourseTestCase):
Expand Down Expand Up @@ -123,6 +129,7 @@ def test_create_dto_recursive_returns_for_leaf_node(self):
'brokenLinks': ['broken_link_1', 'broken_link_2'],
'lockedLinks': ['locked_link'],
'externalForbiddenLinks': ['forbidden_link_1'],
'previousRunLinks': [],
}
]
}
Expand Down Expand Up @@ -181,6 +188,7 @@ def test_create_dto_recursive_returns_for_full_tree(self):
'brokenLinks': ['broken_link_1', 'broken_link_2'],
'lockedLinks': ['locked_link'],
'externalForbiddenLinks': ['forbidden_link_1'],
'previousRunLinks': [],
}
]
}
Expand Down Expand Up @@ -295,3 +303,145 @@ def test_sorts_sections_correctly(self, mock_modulestore):
]

assert result["LinkCheckOutput"]["sections"] == expected_sections

def test_prev_run_link_detection(self):
"""Test the core logic of separating previous run links from regular links."""

previous_course_key = CourseKey.from_string(
"course-v1:edX+DemoX+Demo_Course_2023"
)

test_cases = [
(f"/courses/{previous_course_key}/info", True),
(f"/courses/{previous_course_key}/courseware", True),
(f"/courses/{str(previous_course_key).upper()}/page", True),
# Should NOT match
("/courses/course-v1:edX+DemoX+Demo_Course_2024/info", False),
("/static/image.png", False),
("/assets/courseware/file.pdf", False),
("", False),
(" ", False),
]

for url, expected_match in test_cases:
with self.subTest(url=url, expected=expected_match):
result = contains_previous_course_reference(url, previous_course_key)
self.assertEqual(
result,
expected_match,
f"URL '{url}' should {'match' if expected_match else 'not match'} previous course",
)

def test_enhanced_url_detection_edge_cases(self):
"""Test edge cases for enhanced URL detection."""

test_cases = [
("", []), # Empty content
("No URLs here", []), # Content without URLs
(
"Visit https://example.com today!",
["https://example.com"],
), # URL in text
('href="#anchor"', []), # Should exclude fragments
('src="data:image/png;base64,123"', []), # Should exclude data URLs
(
"Multiple URLs: http://site1.com and https://site2.com",
["http://site1.com", "https://site2.com"],
), # Multiple URLs
(
"URL with params: https://example.com/page?param=value&other=123",
["https://example.com/page?param=value&other=123"],
), # URL with parameters
]

for content, expected_urls in test_cases:
with self.subTest(content=content):
urls = extract_content_URLs_from_course(content)
for expected_url in expected_urls:
self.assertIn(
expected_url,
urls,
f"Should find '{expected_url}' in content: {content}",
)

def test_course_updates_and_custom_pages_structure(self):
"""Test that course_updates and custom_pages are properly structured in the response."""

json_content = [
# Regular course content
[
"course-v1:Test+Course+2024+type@html+block@content1",
"http://content-link.com",
"broken",
],
[
"course-v1:Test+Course+2024+type@vertical+block@unit1",
"http://unit-link.com",
"locked",
],
# Course updates
[
"course-v1:Test+Course+2024+type@course_info+block@updates",
"http://update1.com",
"broken",
],
[
"course-v1:Test+Course+2024+type@course_info+block@updates",
"http://update2.com",
"locked",
],
# Handouts (should be merged into course_updates)
[
"course-v1:Test+Course+2024+type@course_info+block@handouts",
"http://handout.com",
"broken",
],
# Custom pages (static tabs)
[
"course-v1:Test+Course+2024+type@static_tab+block@page1",
"http://page1.com",
"broken",
],
[
"course-v1:Test+Course+2024+type@static_tab+block@page2",
"http://page2.com",
"external-forbidden",
],
]

with mock.patch(
"cms.djangoapps.contentstore.core.course_optimizer_provider._generate_links_descriptor_for_content"
) as mock_content, mock.patch(
"cms.djangoapps.contentstore.core.course_optimizer_provider.modulestore"
) as mock_modulestore:

mock_content.return_value = {"sections": []}
mock_course = self.mock_course
mock_tab1 = StaticTab(name="Page1", url_slug="page1")
mock_tab2 = StaticTab(name="Page2", url_slug="page2")
mock_course.tabs = [mock_tab1, mock_tab2]
mock_course.id = CourseKey.from_string("course-v1:Test+Course+2024")
mock_modulestore.return_value.get_course.return_value = mock_course

course_key = CourseKey.from_string("course-v1:Test+Course+2024")
result = generate_broken_links_descriptor(
json_content, self.user, course_key
)

# Verify top-level structure
self.assertIn("sections", result)
self.assertIn("course_updates", result)
self.assertIn("custom_pages", result)
self.assertNotIn("handouts", result)

# Course updates should include both updates and handouts
self.assertGreaterEqual(
len(result["course_updates"]),
1,
"Should have course updates/handouts",
)

# Custom pages should have custom pages data
self.assertGreaterEqual(
len(result["custom_pages"]), 1, "Should have custom pages"
)
3 changes: 3 additions & 0 deletions cms/djangoapps/contentstore/rest_api/v0/serializers/course_optimizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ class LinkCheckBlockSerializer(serializers.Serializer):
brokenLinks = serializers.ListField(required=False)
lockedLinks = serializers.ListField(required=False)
externalForbiddenLinks = serializers.ListField(required=False)
previousRunLinks = serializers.ListField(required=False)


class LinkCheckUnitSerializer(serializers.Serializer):
Expand All @@ -39,6 +40,8 @@ class LinkCheckSectionSerializer(serializers.Serializer):
class LinkCheckOutputSerializer(serializers.Serializer):
""" Serializer for broken links output model data """
sections = LinkCheckSectionSerializer(many=True)
course_updates = LinkCheckBlockSerializer(many=True, required=False)
custom_pages = LinkCheckBlockSerializer(many=True, required=False)


class LinkCheckSerializer(serializers.Serializer):
Expand Down
96 changes: 64 additions & 32 deletions cms/djangoapps/contentstore/rest_api/v0/views/course_optimizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,53 +71,49 @@ class LinkCheckStatusView(DeveloperErrorViewMixin, APIView):
)
def get(self, request: Request, course_id: str):
"""
GET handler to return the status of the link_check task from UserTaskStatus.
If no task has been started for the course, return 'Uninitiated'.
If link_check task was successful, an output result is also returned.
**Use Case**

For reference, the following status are in UserTaskStatus:
'Pending', 'In Progress' (sent to frontend as 'In-Progress'),
'Succeeded', 'Failed', 'Canceled', 'Retrying'
This function adds a status for when status from UserTaskStatus is None:
'Uninitiated'
GET handler to return the status of the link_check task from UserTaskStatus.
If no task has been started for the course, return 'Uninitiated'.
If link_check task was successful, an output result is also returned.

For reference, the following status are in UserTaskStatus:
'Pending', 'In Progress' (sent to frontend as 'In-Progress'),
'Succeeded', 'Failed', 'Canceled', 'Retrying'
This function adds a status for when status from UserTaskStatus is None:
'Uninitiated'

**Example Request**

GET /api/contentstore/v0/link_check_status/{course_id}

**Example Response**

```json
{
"LinkCheckStatus": "Succeeded",
"LinkCheckCreatedAt": "2025-02-05T14:32:01.294587Z",
"LinkCheckOutput": {
sections: [
"sections": [
{
id: <string>,
displayName: <string>,
subsections: [
"id": <string>,
"displayName": <string>,
"subsections": [
{
id: <string>,
displayName: <string>,
units: [
"id": <string>,
"displayName": <string>,
"units": [
{
id: <string>,
displayName: <string>,
blocks: [
"id": <string>,
"displayName": <string>,
"blocks": [
{
id: <string>,
url: <string>,
brokenLinks: [
<string>,
<string>,
<string>,
...,
],
lockedLinks: [
<string>,
<string>,
<string>,
...,
],
"id": <string>,
"url": <string>,
"brokenLinks": [<string>, ...],
"lockedLinks": [<string>, ...],
"externalForbiddenLinks": [<string>, ...],
"previousRunLinks": [<string>, ...]
},
{ <another block> },
],
Expand All @@ -130,6 +126,42 @@ def get(self, request: Request, course_id: str):
},
{ <another section> },
],
"course_updates": [
{
"id": <string>,
"displayName": <string>,
"url": <string>,
"brokenLinks": [<string>, ...],
"lockedLinks": [<string>, ...],
"externalForbiddenLinks": [<string>, ...],
"previousRunLinks": [<string>, ...]
},
...,
{ <another course-updates> },
...,
{
"id": <string>,
"displayName": "handouts",
"url": <string>,
"brokenLinks": [<string>, ...],
"lockedLinks": [<string>, ...],
"externalForbiddenLinks": [<string>, ...],
"previousRunLinks": [<string>, ...]
}
],
"custom_pages": [
{
"id": <string>,
"displayName": <string>,
"url": <string>,
"brokenLinks": [<string>, ...],
"lockedLinks": [<string>, ...],
"externalForbiddenLinks": [<string>, ...],
"previousRunLinks": [<string>, ...]
},
...,
{ <another page> },
]
},
}
"""
Expand Down
8 changes: 8 additions & 0 deletions cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class CourseWaffleFlagsSerializer(serializers.Serializer):
enable_course_optimizer = serializers.SerializerMethodField()
use_react_markdown_editor = serializers.SerializerMethodField()
use_video_gallery_flow = serializers.SerializerMethodField()
enable_course_optimizer_check_prev_run_links = serializers.SerializerMethodField()

def get_course_key(self):
"""
Expand Down Expand Up @@ -167,3 +168,10 @@ def get_use_video_gallery_flow(self, obj):
Method to get the use_video_gallery_flow waffle flag
"""
return toggles.use_video_gallery_flow()

def get_enable_course_optimizer_check_prev_run_links(self, obj):
"""
Method to get the enable_course_optimizer_check_prev_run_links waffle flag
"""
course_key = self.get_course_key()
return toggles.enable_course_optimizer_check_prev_run_links(course_key)
Loading
Loading