From 21e3a5536ea19a28fe042942ddb9feef6d60727b Mon Sep 17 00:00:00 2001 From: Kasper Brandt Date: Mon, 15 Aug 2016 15:05:53 +0200 Subject: [PATCH 1/2] [#2329] Add 'offset' pagination for /api/v1/ --- akvo/rest/pagination.py | 109 ++++++++++++++++++++++++++++++++++++++ akvo/rest/renderers.py | 13 +++-- akvo/rest/viewsets.py | 72 ++++++++++++++++++++++++- akvo/settings/30-rsr.conf | 1 + 4 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 akvo/rest/pagination.py diff --git a/akvo/rest/pagination.py b/akvo/rest/pagination.py new file mode 100644 index 0000000000..68ad61c8c4 --- /dev/null +++ b/akvo/rest/pagination.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- + +# Akvo RSR is covered by the GNU Affero General Public License. +# See more details in the license.txt file located at the root folder of the Akvo RSR module. +# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. + +from django.utils.six.moves.urllib import parse as urlparse + +from rest_framework import pagination, serializers +from rest_framework.templatetags.rest_framework import replace_query_param + + +def remove_query_param(url, key): + """ + Given a URL and a key/val pair, remove an item in the query + parameters of the URL, and return the new URL. + """ + (scheme, netloc, path, query, fragment) = urlparse.urlsplit(url) + query_dict = urlparse.parse_qs(query, keep_blank_values=True) + query_dict.pop(key, None) + query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True) + return urlparse.urlunsplit((scheme, netloc, path, query, fragment)) + + +class NextPageField(serializers.Field): + """ + Field that returns a link to the next page in paginated results. + """ + page_field = 'page' + offset_field = 'offset' + + def to_native(self, value): + request = self.context.get('request') + + if request and '/api/v1/' not in request.path: + if not value.has_next(): + return None + page = value.next_page_number() + url = request and request.build_absolute_uri() or '' + new_url = remove_query_param(url, self.offset_field) + return replace_query_param(new_url, self.page_field, page) + + elif request: + if not value.has_next(): + return None + try: + offset = int(request.GET.get('offset') or 0) + except ValueError: + offset = 0 + new_offset = offset + value.object_list.count() + url = request and request.build_absolute_uri() or '' + if new_offset > 0: + return replace_query_param(url, self.offset_field, new_offset) + else: + return remove_query_param(url, self.offset_field) + + +class PreviousPageField(serializers.Field): + """ + Field that returns a link to the next page in paginated results. + """ + page_field = 'page' + offset_field = 'offset' + + def to_native(self, value): + request = self.context.get('request') + + if request and '/api/v1/' not in request.path: + if not value.has_previous(): + return None + page = value.previous_page_number() + url = request and request.build_absolute_uri() or '' + new_url = remove_query_param(url, self.offset_field) + return replace_query_param(new_url, self.page_field, page) + + elif request: + if not value.has_previous(): + return None + try: + offset = int(request.GET.get('offset') or 0) + except ValueError: + offset = 0 + new_offset = offset - value.object_list.count() + url = request and request.build_absolute_uri() or '' + if new_offset > 0: + return replace_query_param(url, self.offset_field, new_offset) + else: + return remove_query_param(url, self.offset_field) + + +class OffsetPageField(serializers.Field): + """ + Field that returns the offset. + """ + def to_native(self, value): + request = self.context.get('request') + try: + return int(request.GET.get('offset') or 0) + except ValueError: + return 0 + + +class CustomPaginationSerializer(pagination.PaginationSerializer): + offset = OffsetPageField(source='*') + next = NextPageField(source='*') + previous = PreviousPageField(source='*') + + def __init__(self, *args, **kwargs): + super(CustomPaginationSerializer, self).__init__(*args, **kwargs) diff --git a/akvo/rest/renderers.py b/akvo/rest/renderers.py index c8fda1e25f..3558f5fe44 100644 --- a/akvo/rest/renderers.py +++ b/akvo/rest/renderers.py @@ -68,6 +68,7 @@ def _convert_data_for_tastypie(request, view, data): response_data['meta'] = { 'limit': view.get_paginate_by(), 'total_count': data.get('count'), + 'offset': data.get('offset'), 'next': _remove_domain(request, data.get('next')), 'previous': _remove_domain(request, data.get('previous')), } @@ -95,12 +96,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None): request = renderer_context.get('request') if '/api/v1/' in request.path: - if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']): + if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']): # Paginated result data = _convert_data_for_tastypie(request, renderer_context['view'], data) else: # Non-paginated result data = _rename_fields([data])[0] + elif 'offset' in data.keys(): + data.pop('offset') return super(CustomHTMLRenderer, self).render(data, accepted_media_type, renderer_context) @@ -125,12 +128,14 @@ def render(self, data, accepted_media_type=None, renderer_context=None): request = renderer_context.get('request') if '/api/v1/' in request.path: - if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']): + if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']): # Paginated result data = _convert_data_for_tastypie(request, renderer_context['view'], data) else: # Non-paginated result data = _rename_fields([data])[0] + elif 'offset' in data.keys(): + data.pop('offset') return super(CustomJSONRenderer, self).render(data, accepted_media_type, renderer_context) @@ -190,7 +195,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): xml = SimplerXMLGenerator(stream, self.charset) xml.startDocument() - if all(k in data.keys() for k in ['count', 'next', 'previous', 'results']): + if all(k in data.keys() for k in ['count', 'next', 'previous', 'offset', 'results']): # Paginated result xml.startElement("response", {}) data = _convert_data_for_tastypie(request, renderer_context['view'], data) @@ -204,5 +209,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): xml.endDocument() return stream.getvalue() + elif 'offset' in data.keys(): + data.pop('offset') return XMLRenderer().render(data, accepted_media_type, renderer_context) diff --git a/akvo/rest/viewsets.py b/akvo/rest/viewsets.py index 1d485b82df..2fa7f6f32d 100644 --- a/akvo/rest/viewsets.py +++ b/akvo/rest/viewsets.py @@ -7,6 +7,8 @@ from django.db.models.fields import FieldDoesNotExist from django.db.models.fields.related import ForeignObject from django.core.exceptions import FieldError +from django.core.paginator import InvalidPage +from django.http import Http404 from akvo.rest.models import TastyTokenAuthentication @@ -14,6 +16,8 @@ from .filters import RSRGenericFilterBackend +import warnings + class SafeMethodsPermissions(permissions.DjangoObjectPermissions): """ @@ -36,6 +40,70 @@ class BaseRSRViewSet(viewsets.ModelViewSet): filter_backends = (filters.OrderingFilter, RSRGenericFilterBackend,) ordering_fields = '__all__' + def paginate_queryset(self, queryset, page_size=None): + """ + Paginate a queryset if required, either returning a page object, + or `None` if pagination is not configured for this view. + """ + if '/rest/v1/' in self.request.path: + return super(BaseRSRViewSet, self).paginate_queryset(queryset, page_size) + + deprecated_style = False + if page_size is not None: + warnings.warn('The `page_size` parameter to `paginate_queryset()` ' + 'is deprecated. ' + 'Note that the return style of this method is also ' + 'changed, and will simply return a page object ' + 'when called without a `page_size` argument.', + DeprecationWarning, stacklevel=2) + deprecated_style = True + else: + # Determine the required page size. + # If pagination is not configured, simply return None. + page_size = self.get_paginate_by() + if not page_size: + return None + + if not self.allow_empty: + warnings.warn( + 'The `allow_empty` parameter is deprecated. ' + 'To use `allow_empty=False` style behavior, You should override ' + '`get_queryset()` and explicitly raise a 404 on empty querysets.', + DeprecationWarning, stacklevel=2 + ) + + paginator = self.paginator_class(queryset, page_size, + allow_empty_first_page=self.allow_empty) + offset_kwarg = self.kwargs.get('offset') + offset_query_param = self.request.QUERY_PARAMS.get('offset') + + try: + offset = int(offset_kwarg or offset_query_param or 0) + except ValueError: + raise Http404(_("Offset cannot be converted to an int.")) + + page = int(offset / page_size) + 1 + + try: + page_number = paginator.validate_number(page) + except InvalidPage: + if page == 'last': + page_number = paginator.num_pages + else: + raise Http404(_("Page is not 'last', nor can it be converted to an int.")) + try: + page = paginator.page(page_number) + except InvalidPage as exc: + error_format = _('Invalid page (%(page_number)s): %(message)s') + raise Http404(error_format % { + 'page_number': page_number, + 'message': str(exc) + }) + + if deprecated_style: + return (paginator, page, page.object_list, page.has_other_pages()) + return page + def get_queryset(self): def django_filter_filters(request): @@ -45,8 +113,8 @@ def django_filter_filters(request): # query string keys reserved by the RSRGenericFilterBackend qs_params = ['filter', 'exclude', 'select_related', 'prefetch_related', ] # query string keys used by core DRF, OrderingFilter and Akvo custom views - exclude_params = ['limit', 'format', 'page', 'ordering', 'partner_type', 'sync_owner', - 'reporting_org', ] + exclude_params = ['limit', 'format', 'page', 'offset', 'ordering', 'partner_type', + 'sync_owner', 'reporting_org', ] filters = {} for key in request.QUERY_PARAMS.keys(): if key not in qs_params + exclude_params and not key.startswith('image_thumb_'): diff --git a/akvo/settings/30-rsr.conf b/akvo/settings/30-rsr.conf index aa39c581b6..423b5290a2 100644 --- a/akvo/settings/30-rsr.conf +++ b/akvo/settings/30-rsr.conf @@ -70,6 +70,7 @@ REST_FRAMEWORK = { 'rest_framework.parsers.JSONParser', 'rest_framework.parsers.XMLParser', ), + 'DEFAULT_PAGINATION_SERIALIZER_CLASS': 'akvo.rest.pagination.CustomPaginationSerializer', # Harmonize datetime format across serializer formats 'DATETIME_FORMAT': 'iso-8601', 'DEFAULT_FILTER_BACKENDS': ( From 622e0a29eab509bde0bf87a6ca8b7b4d7282890d Mon Sep 17 00:00:00 2001 From: Kasper Brandt Date: Mon, 15 Aug 2016 15:54:45 +0200 Subject: [PATCH 2/2] [#2329] Fix bug in URL renderer --- akvo/rest/renderers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/akvo/rest/renderers.py b/akvo/rest/renderers.py index 3558f5fe44..d210e468d7 100644 --- a/akvo/rest/renderers.py +++ b/akvo/rest/renderers.py @@ -7,6 +7,8 @@ from django.utils import six from django.utils.xmlutils import SimplerXMLGenerator +from six import string_types + from rest_framework.compat import StringIO, smart_text from rest_framework.renderers import BaseRenderer, BrowsableAPIRenderer, JSONRenderer, XMLRenderer @@ -43,10 +45,15 @@ def _rename_fields(results): if isinstance(result[object_id], list): result_list, new_list = result.pop(object_id), [] for list_item in result_list: - new_list.append(id_to_api[object_id].format(str(list_item))) + if isinstance(list_item, string_types) and '/api/v1/' in list_item: + new_list.append(list_item) + else: + new_list.append(id_to_api[object_id].format(str(list_item))) result[object_id] = new_list else: - result[object_id] = id_to_api[object_id].format(str(result.pop(object_id))) + if not (isinstance(result[object_id], string_types) and + '/api/v1' in result[object_id]): + result[object_id] = id_to_api[object_id].format(str(result.pop(object_id))) return results