diff --git a/api/ceramic_cache/admin.py b/api/ceramic_cache/admin.py index 0184f5f25..d858d57ea 100644 --- a/api/ceramic_cache/admin.py +++ b/api/ceramic_cache/admin.py @@ -98,7 +98,8 @@ def clean_csv_file(self): Validate the content of the CSV file """ csv_file = self.cleaned_data["csv_file"] - csv_data = csv_file.read().decode("utf-8") + # using 'utf-8-sig' will also handle the case where the file is saved with a BOM (byte order mark - \ufeff) + csv_data = csv_file.read().decode("utf-8-sig") csv_reader = csv.DictReader(StringIO(csv_data)) line = 0 @@ -140,7 +141,9 @@ def save_model(self, request, obj: RevocationList, form, change): super().save_model(request, obj, form, change) # If saving any of the objects below fails, we expect to roll back - csv_reader = csv.DictReader(obj.csv_file.open("rt")) + # using 'utf-8-sig' will also handle the case where the file is saved with a BOM (byte order mark - \ufeff) + csv_data = obj.csv_file.open("rb").read().decode("utf-8-sig") + csv_reader = csv.DictReader(StringIO(csv_data)) revocation_item_list = [] for revocation_item in csv_reader: proof_value = revocation_item["proof_value"] @@ -249,7 +252,8 @@ def clean_csv_file(self): Validate the content of the CSV file """ csv_file = self.cleaned_data["csv_file"] - csv_data = csv_file.read().decode("utf-8") + # using 'utf-8-sig' will also handle the case where the file is saved with a BOM (byte order mark - \ufeff) + csv_data = csv_file.read().decode("utf-8-sig") csv_reader = csv.DictReader(StringIO(csv_data)) line = 0 @@ -267,12 +271,16 @@ def clean_csv_file(self): db_ban_item.full_clean() except ValueError as e: - raise ValidationError(f"Failed to validate line {line}, {e}") from e + raise ValidationError( + f"Failed to validate line {line}, {e}.\nDetected {len(ban_item)} columns: '{[c for c in ban_item.keys()]}' in uploaded CSV file" + ) from e except ValidationError as e: - raise ValidationError(f"Failed to validate line {line}, {e}") from e + raise ValidationError( + f"Failed to validate line {line}, {e}.\nDetected {len(ban_item)} columns: '{[c for c in ban_item.keys()]}' in uploaded CSV file" + ) from e except Exception as e: raise ValidationError( - f"Failed to validate line {line} (unknown error), {e}" + f"Failed to validate line {line} (unknown error), {e}.\nDetected {len(ban_item)} columns: '{[c for c in ban_item.keys()]}' in uploaded CSV file" ) from e return csv_file @@ -293,7 +301,9 @@ def save_model(self, request, obj: BanList, form, change): super().save_model(request, obj, form, change) # If saving any of the objects below fails, we expect to roll back - csv_reader = csv.DictReader(obj.csv_file.open("rt")) + # using 'utf-8-sig' will also handle the case where the file is saved with a BOM (byte order mark - \ufeff) + csv_data = obj.csv_file.open("rb").read().decode("utf-8-sig") + csv_reader = csv.DictReader(StringIO(csv_data)) ban_item_list = [] for ban_item in csv_reader: db_ban_item = Ban( diff --git a/api/scorer/settings/storages.py b/api/scorer/settings/storages.py index 925d8c0fe..32b150042 100644 --- a/api/scorer/settings/storages.py +++ b/api/scorer/settings/storages.py @@ -8,6 +8,7 @@ STORAGES = { "default": { "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": {"file_overwrite": False}, }, "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",