Validate XLS rows count before exporting.

Author: hsmett

CHANGELOG.txt

@@ -9,6 +9,8 @@
     # A visitor mode has been added; from any list-view, you can visit all the related detail-views the one after the other.
       (button "Enter the exploration mode" in list-views).
     # The link in the list-views selectors are now opened in other tabs automatically.
+    # Prevent an error when reaching 65535 rows during an .XLS file export, which is a hard limit defined by the .XLS
+      format specs. Display a comprehensive error message prior to the export.
     # In forms :
         - The custom fields with ENUM/MULTI_ENUM type are now using the Select2 combobox with lazy loading & search.
         - In the form for entity-filters :

creme/creme_core/backends/base.py

@@ -71,3 +71,6 @@ def save(self, filename: str, user):
               instance of <django.contrib.auth.get_user_model()>.
         """
         raise NotImplementedError
+
+    def validate(self, **kwargs):
+        pass

creme/creme_core/backends/xls_export.py

@@ -21,8 +21,10 @@
 from django.conf import settings
 from django.http import HttpResponseRedirect
 from django.template.defaultfilters import slugify
+from django.utils.translation import gettext
 from django.utils.translation import gettext_lazy as _
 
+from ..core.exceptions import ConflictError
 from ..models import FileRef
 from ..utils.file_handling import FileCreator
 from ..utils.xlwt_utils import XlwtWriter
@@ -34,6 +36,7 @@ class XLSExportBackend(ExportBackend):
     verbose_name = _('XLS File')
     help_text = ''
     dir_parts = ('xls',)  # Sub-directory under settings.MEDIA_ROOT
+    max_row_index = 65535
 
     def __init__(self, encoding='utf-8'):
         super().__init__()
@@ -56,3 +59,13 @@ def save(self, filename, user):
 
     def writerow(self, row):
         self.writer.writerow(row)
+
+    def validate(self, **kwargs):
+        total_count = kwargs.get("total_count")
+        if total_count and total_count > self.max_row_index:
+            raise ConflictError(
+                gettext(
+                    "The Microsoft Excel 97–2003 Workbook (.xls) format has a limit of"
+                    " {count} rows. Use a CSV format instead.".format(count=self.max_row_index)
+                )
+            )

creme/creme_core/locale/fr/LC_MESSAGES/django.po

@@ -75,6 +75,13 @@ msgstr ""
 msgid "XLS File"
 msgstr "Fichier XLS"
 
+#, python-brace-format
+msgid ""
+"The Microsoft Excel 97–2003 Workbook (.xls) format has a limit of {count} "
+"rows. Use a CSV format instead."
+msgstr "Le format Microsoft Excel 97–2003 Workbook (.xls) est limité à {count} lignes. "
+"Utilisez un format CSV à la place."
+
 msgid ""
 "XLS is a file extension for a spreadsheet file format created by Microsoft "
 "for use with Microsoft Excel (Excel 97-2003 Workbook)."

creme/creme_core/tests/test_backends.py

@@ -2,6 +2,8 @@
 from creme.creme_core.backends.csv_import import CSVImportBackend
 from creme.creme_core.backends.xls_import import XLSImportBackend
 
+from ..backends.xls_export import XLSExportBackend
+from ..core.exceptions import ConflictError
 from .base import CremeTestCase
 
 
@@ -56,3 +58,14 @@ def test_registration_errors03(self):
 
         with self.assertRaises(registry.InvalidClass):
             registry.get_backend_class(CSVImportBackend.id)
+
+
+class XLSExportBackendTestCase(CremeTestCase):
+    def test_validate_total_count__lte_max(self):
+        backend = XLSExportBackend()
+        backend.validate(total_count=XLSExportBackend.max_row_index)
+
+    def test_validate_total_count__gt_max(self):
+        backend = XLSExportBackend()
+        with self.assertRaises(ConflictError):
+            backend.validate(total_count=XLSExportBackend.max_row_index + 1)

creme/creme_core/tests/views/test_mass_export.py

@@ -46,9 +46,24 @@
 from creme.creme_core.utils.queries import QSerializer
 from creme.creme_core.utils.xlrd_utils import XlrdReader
 
+from ...backends import export_backend_registry
+from ...backends.base import ExportBackend
+from ...core.exceptions import ConflictError
 from .base import ViewsTestCase
 
 
+class TestExportBackend(ExportBackend):
+    id: str = 'test_backend_validator'
+    verbose_name: str = 'test_backend_validator'
+    help_text: str = 'test_backend_validator'
+
+    def writerow(self, row):
+        pass
+
+    def validate(self, **kwargs):
+        raise ConflictError("TestExportBackend.validate fail")
+
+
 class MassExportViewsTestCase(ViewsTestCase):
     @classmethod
     def setUpClass(cls):
@@ -529,6 +544,14 @@ def test_extra_filter(self):
         # Error
         self.assertGET(400, self._build_contact_dl_url(extra_q='[123]'))
 
+    def test_backend_validator(self):
+        self.login_as_root()
+        self.assertIsNone(export_backend_registry.get_backend_class(TestExportBackend.id))
+        export_backend_registry._backend_classes[TestExportBackend.id] = TestExportBackend
+        response = self.assertGET409(self._build_contact_dl_url(doc_type=TestExportBackend.id))
+        self.assertContains(response, 'TestExportBackend.validate fail', status_code=409)
+        export_backend_registry._backend_classes = None
+
     def test_list_view_export_with_filter01(self):
         # user = self.login()
         user = self.login_as_root_and_get()

creme/creme_core/views/mass_export.py

@@ -227,6 +227,8 @@ def get(self, request, *args, **kwargs):
             if use_distinct:
                 entities_qs = entities_qs.distinct()
 
+            writer.validate(total_count=entities_qs.count())
+
             paginator = self.get_paginator(queryset=entities_qs, ordering=ordering)
 
             total_count = 0