Merge pull request #2332 from akvo/#2329-api-offset

[#2329] Offset for 'old' API

akvo/rest/pagination.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+
+# Akvo RSR is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+
+from django.utils.six.moves.urllib import parse as urlparse
+
+from rest_framework import pagination, serializers
+from rest_framework.templatetags.rest_framework import replace_query_param
+
+
+def remove_query_param(url, key):
+    """
+    Given a URL and a key/val pair, remove an item in the query
+    parameters of the URL, and return the new URL.
+    """
+    (scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
+    query_dict = urlparse.parse_qs(query, keep_blank_values=True)
+    query_dict.pop(key, None)
+    query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True)
+    return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
+
+
+class NextPageField(serializers.Field):
+    """
+    Field that returns a link to the next page in paginated results.
+    """
+    page_field = 'page'
+    offset_field = 'offset'
+
+    def to_native(self, value):
+        request = self.context.get('request')
+
+        if request and '/api/v1/' not in request.path:
+            if not value.has_next():
+                return None
+            page = value.next_page_number()
+            url = request and request.build_absolute_uri() or ''
+            new_url = remove_query_param(url, self.offset_field)
+            return replace_query_param(new_url, self.page_field, page)
+
+        elif request:
+            if not value.has_next():
+                return None
+            try:
+                offset = int(request.GET.get('offset') or 0)
+            except ValueError:
+                offset = 0
+            new_offset = offset + value.object_list.count()
+            url = request and request.build_absolute_uri() or ''
+            if new_offset > 0:
+                return replace_query_param(url, self.offset_field, new_offset)
+            else:
+                return remove_query_param(url, self.offset_field)
+
+
+class PreviousPageField(serializers.Field):
+    """
+    Field that returns a link to the next page in paginated results.
+    """
+    page_field = 'page'
+    offset_field = 'offset'
+
+    def to_native(self, value):
+        request = self.context.get('request')
+
+        if request and '/api/v1/' not in request.path:
+            if not value.has_previous():
+                return None
+            page = value.previous_page_number()
+            url = request and request.build_absolute_uri() or ''
+            new_url = remove_query_param(url, self.offset_field)
+            return replace_query_param(new_url, self.page_field, page)
+
+        elif request:
+            if not value.has_previous():
+                return None
+            try:
+                offset = int(request.GET.get('offset') or 0)
+            except ValueError:
+                offset = 0
+            new_offset = offset - value.object_list.count()
+            url = request and request.build_absolute_uri() or ''
+            if new_offset > 0:
+                return replace_query_param(url, self.offset_field, new_offset)
+            else:
+                return remove_query_param(url, self.offset_field)
+
+
+class OffsetPageField(serializers.Field):
+    """
+    Field that returns the offset.
+    """
+    def to_native(self, value):
+        request = self.context.get('request')
+        try:
+            return int(request.GET.get('offset') or 0)
+        except ValueError:
+            return 0
+
+
+class CustomPaginationSerializer(pagination.PaginationSerializer):
+    offset = OffsetPageField(source='*')
+    next = NextPageField(source='*')
+    previous = PreviousPageField(source='*')
+
+    def __init__(self, *args, **kwargs):
+        super(CustomPaginationSerializer, self).__init__(*args, **kwargs)

akvo/rest/renderers.py

@@ -7,6 +7,8 @@
 from django.utils import six
 from django.utils.xmlutils import SimplerXMLGenerator
 
+from six import string_types
+
 from rest_framework.compat import StringIO, smart_text
 from rest_framework.renderers import BaseRenderer, BrowsableAPIRenderer, JSONRenderer, XMLRenderer
 
@@ -43,10 +45,15 @@ def _rename_fields(results):
                 if isinstance(result[object_id], list):
                     result_list, new_list = result.pop(object_id), []
                     for list_item in result_list:
-                        new_list.append(id_to_api[object_id].format(str(list_item)))
+                        if isinstance(list_item, string_types) and '/api/v1/' in list_item:
+                            new_list.append(list_item)
+                        else:
+                            new_list.append(id_to_api[object_id].format(str(list_item)))
                     result[object_id] = new_list
                 else:
-                    result[object_id] = id_to_api[object_id].format(str(result.pop(object_id)))
+                    if not (isinstance(result[object_id], string_types) and
+                            '/api/v1' in result[object_id]):
+                        result[object_id] = id_to_api[object_id].format(str(result.pop(object_id)))
     return results
 
 
@@ -68,6 +75,7 @@ def _convert_data_for_tastypie(request, view, data):
     response_data['meta'] = {
         'limit': view.get_paginate_by(),
         'total_count': data.get('count'),
+        'offset': data.get('offset'),
         'next': _remove_domain(request, data.get('next')),
         'previous': _remove_domain(request, data.get('previous')),
     }
@@ -95,12 +103,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
         request = renderer_context.get('request')
 
         if '/api/v1/' in request.path:
-            if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']):
+            if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']):
                 # Paginated result
                 data = _convert_data_for_tastypie(request, renderer_context['view'], data)
             else:
                 # Non-paginated result
                 data = _rename_fields([data])[0]
+        elif 'offset' in data.keys():
+            data.pop('offset')
 
         return super(CustomHTMLRenderer, self).render(data, accepted_media_type, renderer_context)
 
@@ -125,12 +135,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
         request = renderer_context.get('request')
 
         if '/api/v1/' in request.path:
-            if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']):
+            if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']):
                 # Paginated result
                 data = _convert_data_for_tastypie(request, renderer_context['view'], data)
             else:
                 # Non-paginated result
                 data = _rename_fields([data])[0]
+        elif 'offset' in data.keys():
+            data.pop('offset')
 
         return super(CustomJSONRenderer, self).render(data, accepted_media_type, renderer_context)
 
@@ -190,7 +202,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
             xml = SimplerXMLGenerator(stream, self.charset)
             xml.startDocument()
 
-            if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']):
+            if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']):
                 # Paginated result
                 xml.startElement("response", {})
                 data = _convert_data_for_tastypie(request, renderer_context['view'], data)
@@ -204,5 +216,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None):
 
             xml.endDocument()
             return stream.getvalue()
+        elif 'offset' in data.keys():
+            data.pop('offset')
 
         return XMLRenderer().render(data, accepted_media_type, renderer_context)

akvo/rest/viewsets.py

@@ -7,13 +7,17 @@
 from django.db.models.fields import FieldDoesNotExist
 from django.db.models.fields.related import ForeignObject
 from django.core.exceptions import FieldError
+from django.core.paginator import InvalidPage
+from django.http import Http404
 
 from akvo.rest.models import TastyTokenAuthentication
 
 from rest_framework import authentication, filters, permissions, viewsets
 
 from .filters import RSRGenericFilterBackend
 
+import warnings
+
 
 class SafeMethodsPermissions(permissions.DjangoObjectPermissions):
     """
@@ -36,6 +40,70 @@ class BaseRSRViewSet(viewsets.ModelViewSet):
     filter_backends = (filters.OrderingFilter, RSRGenericFilterBackend,)
     ordering_fields = '__all__'
 
+    def paginate_queryset(self, queryset, page_size=None):
+        """
+        Paginate a queryset if required, either returning a page object,
+        or `None` if pagination is not configured for this view.
+        """
+        if '/rest/v1/' in self.request.path:
+            return super(BaseRSRViewSet, self).paginate_queryset(queryset, page_size)
+
+        deprecated_style = False
+        if page_size is not None:
+            warnings.warn('The `page_size` parameter to `paginate_queryset()` '
+                          'is deprecated. '
+                          'Note that the return style of this method is also '
+                          'changed, and will simply return a page object '
+                          'when called without a `page_size` argument.',
+                          DeprecationWarning, stacklevel=2)
+            deprecated_style = True
+        else:
+            # Determine the required page size.
+            # If pagination is not configured, simply return None.
+            page_size = self.get_paginate_by()
+            if not page_size:
+                return None
+
+        if not self.allow_empty:
+            warnings.warn(
+                'The `allow_empty` parameter is deprecated. '
+                'To use `allow_empty=False` style behavior, You should override '
+                '`get_queryset()` and explicitly raise a 404 on empty querysets.',
+                DeprecationWarning, stacklevel=2
+            )
+
+        paginator = self.paginator_class(queryset, page_size,
+                                         allow_empty_first_page=self.allow_empty)
+        offset_kwarg = self.kwargs.get('offset')
+        offset_query_param = self.request.QUERY_PARAMS.get('offset')
+
+        try:
+            offset = int(offset_kwarg or offset_query_param or 0)
+        except ValueError:
+            raise Http404(_("Offset cannot be converted to an int."))
+
+        page = int(offset / page_size) + 1
+
+        try:
+            page_number = paginator.validate_number(page)
+        except InvalidPage:
+            if page == 'last':
+                page_number = paginator.num_pages
+            else:
+                raise Http404(_("Page is not 'last', nor can it be converted to an int."))
+        try:
+            page = paginator.page(page_number)
+        except InvalidPage as exc:
+            error_format = _('Invalid page (%(page_number)s): %(message)s')
+            raise Http404(error_format % {
+                'page_number': page_number,
+                'message': str(exc)
+            })
+
+        if deprecated_style:
+            return (paginator, page, page.object_list, page.has_other_pages())
+        return page
+
     def get_queryset(self):
 
         def django_filter_filters(request):
@@ -45,8 +113,8 @@ def django_filter_filters(request):
             # query string keys reserved by the RSRGenericFilterBackend
             qs_params = ['filter', 'exclude', 'select_related', 'prefetch_related', ]
             # query string keys used by core DRF, OrderingFilter and Akvo custom views
-            exclude_params = ['limit', 'format', 'page', 'ordering', 'partner_type', 'sync_owner',
-                              'reporting_org', ]
+            exclude_params = ['limit', 'format', 'page', 'offset', 'ordering', 'partner_type',
+                              'sync_owner', 'reporting_org', ]
             filters = {}
             for key in request.QUERY_PARAMS.keys():
                 if key not in qs_params + exclude_params and not key.startswith('image_thumb_'):

akvo/settings/30-rsr.conf

@@ -70,6 +70,7 @@ REST_FRAMEWORK = {
         'rest_framework.parsers.JSONParser',
         'rest_framework.parsers.XMLParser',
     ),
+    'DEFAULT_PAGINATION_SERIALIZER_CLASS': 'akvo.rest.pagination.CustomPaginationSerializer',
     # Harmonize datetime format across serializer formats
     'DATETIME_FORMAT': 'iso-8601',
     'DEFAULT_FILTER_BACKENDS': (