Improve sitemaps with dynamic priorities

* Old courses have a smaller priority than current courses.
* Add a new base class for sitemaps: `class AplusSitemap`.
  It differentiates the priorities of recent and outdated objects.
* Change some of the priority values that had been used.
  Give chapters the highest priority.
* Remove anchor URLs from the sitemap because sitemaps disallow
  the anchor part (for example, embedded exercises with
  `#chapter-exercise-1`).
* No duplicate URLs in the sitemap.
* For embedded exercises under chapters, add the `noindex` meta tag
  on the full-view exercise page so that it is not indexed.
  Only the chapter page should be indexed since students use
  the chapter view.

Fixes apluslms#975

Author: ihalaij1

aplus/settings.py

@@ -158,6 +158,12 @@
     'aplus_auth',
 )
 
+# Sitemap settings
+##########################################################################
+
+# A course is considered recent or upcoming if it was/is open within this delta (in days) from now.
+SITEMAP_DELTA_DAYS_RECENT_OR_UPCOMING = 180
+
 # Different login options (may override in local_settings.py)
 ##########################################################################
 

course/sitemaps.py

@@ -2,6 +2,7 @@
 from django.urls.base import reverse
 from django.utils import timezone
 
+from lib.sitemaps import AplusSitemap
 from .models import CourseInstance, CourseModule
 
 
@@ -19,32 +20,28 @@ def location(self, item):
         return reverse(item)
 
 
-class InstanceSitemap(sitemaps.Sitemap):
-    priority = 1.0
+class InstanceSitemap(AplusSitemap):
     changefreq = 'daily'
+    base_priority = 0.4
 
     def items(self):
         return CourseInstance.objects.filter(
             view_content_to=CourseInstance.VIEW_ACCESS.PUBLIC,
+            visible_to_students=True,
         )
 
-    def location(self, item):
-        return item.get_display_url()
-
 
-class ModuleSitemap(sitemaps.Sitemap):
-    priority = 0.2
+class ModuleSitemap(AplusSitemap):
     changefreq = 'daily'
+    base_priority = 0.2
 
     def items(self):
         return CourseModule.objects.filter(
             course_instance__view_content_to=CourseInstance.VIEW_ACCESS.PUBLIC,
+            course_instance__visible_to_students=True,
             opening_time__lte=timezone.now(),
         )
 
-    def location(self, item):
-        return item.get_display_url()
-
 
 all_sitemaps = {
     'course_static': CourseStaticViewSitemap,

exercise/sitemaps.py

@@ -1,50 +1,47 @@
-from django.contrib import sitemaps
 from django.urls.base import reverse
 from django.utils import timezone
 
 from course.models import CourseInstance
+from lib.sitemaps import AplusSitemap
 from .models import BaseExercise, CourseChapter, LearningObject
 
 
-class BaseExerciseSitemap(sitemaps.Sitemap):
-    priority = 0.2
+class BaseExerciseSitemap(AplusSitemap):
     changefreq = 'daily'
+    base_priority = 0.5
 
     def items(self):
         return BaseExercise.objects.filter(
             course_module__course_instance__view_content_to=CourseInstance.VIEW_ACCESS.PUBLIC,
+            course_module__course_instance__visible_to_students=True,
             course_module__opening_time__lte=timezone.now(),
-            status__in=[LearningObject.STATUS.READY, LearningObject.STATUS.UNLISTED],
+            status=LearningObject.STATUS.READY,
             audience=LearningObject.AUDIENCE.COURSE_AUDIENCE,
+            parent__isnull=True,
         )
 
-    def location(self, item):
-        return item.get_display_url()
-
 
-class CourseChapterSitemap(sitemaps.Sitemap):
-    priority = 1.0
+class CourseChapterSitemap(AplusSitemap):
     changefreq = 'daily'
 
     def items(self):
         return CourseChapter.objects.filter(
             course_module__course_instance__view_content_to=CourseInstance.VIEW_ACCESS.PUBLIC,
+            course_module__course_instance__visible_to_students=True,
             course_module__opening_time__lte=timezone.now(),
             status__in=[LearningObject.STATUS.READY, LearningObject.STATUS.UNLISTED],
             audience=LearningObject.AUDIENCE.COURSE_AUDIENCE,
         )
 
-    def location(self, item):
-        return item.get_display_url()
-
 
-class TableOfContentsSitemap(sitemaps.Sitemap):
-    priority = 0.2
+class TableOfContentsSitemap(AplusSitemap):
     changefreq = 'monthly'
+    base_priority = 0.2
 
     def items(self):
         return CourseInstance.objects.filter(
             view_content_to=CourseInstance.VIEW_ACCESS.PUBLIC,
+            visible_to_students=True,
         )
 
     def location(self, item):

exercise/templates/exercise/exercise_base.html

@@ -30,6 +30,12 @@
 {% endblock %}
 {% endblock %}
 
+{% block meta %}
+{% if exercise.parent and exercise.status == exercise.STATUS.UNLISTED %}
+<meta name="robots" content="noindex">
+{% endif %}
+{% endblock %}
+
 {% block columns %}
 <div class="{% if exercise.use_wide_column %}col-lg-12{% else %}col-lg-9{% endif %} exercise-column">
 

lib/sitemaps.py

@@ -0,0 +1,45 @@
+import datetime
+
+from django.conf import settings
+from django.contrib import sitemaps
+from django.utils import timezone
+
+from course.models import CourseInstance, CourseModule
+from exercise.models import BaseExercise, CourseChapter
+
+
+class AplusSitemap(sitemaps.Sitemap):
+    # The base priority for pages from old courses.
+    # Pages of recent or upcoming courses have the base priority multiplied.
+    base_priority = 0.5
+
+    def is_recent_or_upcoming(self, item):
+        """
+        Return a boolean value indicating if the given item is open now
+        or if either the item's start date or end date is within a certain time delta from now.
+        """
+        if isinstance(item, CourseInstance):
+            start_date = item.starting_time
+            end_date = item.ending_time
+        elif isinstance(item, CourseModule):
+            start_date = item.opening_time
+            end_date = item.closing_time
+        elif isinstance(item, (BaseExercise, CourseChapter)):
+            start_date = item.course_module.opening_time
+            end_date = item.course_module.closing_time
+        else:
+            return False
+        now = timezone.now()
+        delta = datetime.timedelta(settings.SITEMAP_DELTA_DAYS_RECENT_OR_UPCOMING)
+        is_recent_or_upcoming = (
+            item.is_open(now)
+            or now - delta <= start_date <= now + delta
+            or now - delta <= end_date <= now + delta
+        )
+        return is_recent_or_upcoming
+
+    def location(self, item):
+        return item.get_display_url()
+
+    def priority(self, item):
+        return min(1.0, 2 * self.base_priority) if self.is_recent_or_upcoming(item) else self.base_priority

templates/base.html

@@ -66,6 +66,10 @@
 	{# Put all additional scripts inside this block. #}
 	{% endblock %}
 
+	{% block meta %}
+	{# Put all additional meta data inside this block. #}
+	{% endblock %}
+
 	<!-- Some favicons courtesy of http://realfavicongenerator.net/ -->
 	<link rel="apple-touch-icon" sizes="57x57" href="{{ STATIC_URL }}favicons/apple-touch-icon-57x57.png">
 	<link rel="apple-touch-icon" sizes="114x114" href="{{ STATIC_URL }}favicons/apple-touch-icon-114x114.png">