From c3aae843bcd68815297cc99e51f126b361a227b5 Mon Sep 17 00:00:00 2001 From: Etchegoyen Matthieu Date: Tue, 24 Sep 2024 09:25:12 +0200 Subject: [PATCH] simplify the upload service and use realname or filename at download and display --- .../fixtures/conventions_for_tests.json | 8 +- conventions/services/convention_generator.py | 5 +- conventions/views/conventions.py | 37 ++-- programmes/fixtures/programmes_for_tests.json | 16 +- templates/common/display_files.html | 2 +- upload/fixtures/upload_for_tests.json | 170 ++++++++++++------ .../commands}/__init__.py | 0 .../commands/fix_text_and_files_fields.py | 108 +++++++++++ .../management/commands/fix_upload_files.py | 121 +++++++++++++ .../commands}/s3-backup-errors.log | 0 .../0009_migrate_text_and_files_fields.py | 91 ---------- upload/scripts/fix_upload_files.py | 101 ----------- upload/services.py | 7 - upload/tests/test_services.py | 22 --- upload/views.py | 10 +- 15 files changed, 384 insertions(+), 314 deletions(-) rename upload/{scripts => management/commands}/__init__.py (100%) create mode 100644 upload/management/commands/fix_text_and_files_fields.py create mode 100644 upload/management/commands/fix_upload_files.py rename upload/{scripts => management/commands}/s3-backup-errors.log (100%) delete mode 100644 upload/migrations/0009_migrate_text_and_files_fields.py delete mode 100644 upload/scripts/fix_upload_files.py diff --git a/conventions/fixtures/conventions_for_tests.json b/conventions/fixtures/conventions_for_tests.json index 7c34f0ead..60a923e3d 100644 --- a/conventions/fixtures/conventions_for_tests.json +++ b/conventions/fixtures/conventions_for_tests.json @@ -40,7 +40,7 @@ "date_fin_conventionnement": null, "financement": "PLUS", "fond_propre": null, - "commentaires": "{\"text\": \"this is a test\", \"files\": {\"f7f5151c-6031-4ba4-a041-12666b700def\": {\"uuid\": \"f7f5151c-6031-4ba4-a041-12666b700def\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"a9a47df6-5493-43be-bd21-e86c3d6e79b4\": {\"uuid\": \"a9a47df6-5493-43be-bd21-e86c3d6e79b4\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "commentaires": "{\"text\": \"this is a test\", \"files\": {\"f7f5151c-6031-4ba4-a041-12666b700def\": {\"uuid\": \"f7f5151c-6031-4ba4-a041-12666b700def\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"a9a47df6-5493-43be-bd21-e86c3d6e79b4\": {\"uuid\": \"a9a47df6-5493-43be-bd21-e86c3d6e79b4\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "attached": null, "statut": "1. Projet", "soumis_le": null, @@ -123,7 +123,7 @@ "date_fin_conventionnement": null, "financement": "PLAI", "fond_propre": null, - "commentaires": "{\"text\": \"this is a test\", \"files\": {\"94520361-d943-44ee-a0c0-5ec93c85e2c5\": {\"uuid\": \"94520361-d943-44ee-a0c0-5ec93c85e2c5\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"1a22e492-8d50-4b53-8336-9f925581d8fd\": {\"uuid\": \"1a22e492-8d50-4b53-8336-9f925581d8fd\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "commentaires": "{\"text\": \"this is a test\", \"files\": {\"94520361-d943-44ee-a0c0-5ec93c85e2c5\": {\"uuid\": \"94520361-d943-44ee-a0c0-5ec93c85e2c5\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"1a22e492-8d50-4b53-8336-9f925581d8fd\": {\"uuid\": \"1a22e492-8d50-4b53-8336-9f925581d8fd\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "attached": null, "statut": "1. Projet", "soumis_le": null, @@ -206,7 +206,7 @@ "date_fin_conventionnement": null, "financement": "PLUS", "fond_propre": null, - "commentaires": "{\"text\": \"this is a test\", \"files\": {\"62ddb671-f021-4ae6-8de3-403ba7d80c2c\": {\"uuid\": \"62ddb671-f021-4ae6-8de3-403ba7d80c2c\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"e309ed33-67e0-4fb3-89d9-35db87a9849f\": {\"uuid\": \"e309ed33-67e0-4fb3-89d9-35db87a9849f\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "commentaires": "{\"text\": \"this is a test\", \"files\": {\"62ddb671-f021-4ae6-8de3-403ba7d80c2c\": {\"uuid\": \"62ddb671-f021-4ae6-8de3-403ba7d80c2c\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"e309ed33-67e0-4fb3-89d9-35db87a9849f\": {\"uuid\": \"e309ed33-67e0-4fb3-89d9-35db87a9849f\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "attached": null, "statut": "1. Projet", "soumis_le": null, @@ -289,7 +289,7 @@ "date_fin_conventionnement": null, "financement": "PLAI", "fond_propre": null, - "commentaires": "{\"text\": \"this is a test\", \"files\": {\"11df0962-e377-4133-a128-6a749256c4c8\": {\"uuid\": \"11df0962-e377-4133-a128-6a749256c4c8\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"46caa33d-7e6d-440c-a652-75cab45f4dba\": {\"uuid\": \"46caa33d-7e6d-440c-a652-75cab45f4dba\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "commentaires": "{\"text\": \"this is a test\", \"files\": {\"11df0962-e377-4133-a128-6a749256c4c8\": {\"uuid\": \"11df0962-e377-4133-a128-6a749256c4c8\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"46caa33d-7e6d-440c-a652-75cab45f4dba\": {\"uuid\": \"46caa33d-7e6d-440c-a652-75cab45f4dba\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "attached": null, "statut": "1. Projet", "soumis_le": null, diff --git a/conventions/services/convention_generator.py b/conventions/services/convention_generator.py index 4375c8fb3..95ede1fd1 100644 --- a/conventions/services/convention_generator.py +++ b/conventions/services/convention_generator.py @@ -446,7 +446,10 @@ def get_files_attached(convention): files = UploadedFile.objects.filter(uuid__in=attached_files) for object_file in files: - file = UploadService().get_file(object_file.filepath(convention.uuid)) + file = default_storage.open( + name=object_file.filepath(convention.uuid), + mode="rb", + ) local_path = ( settings.MEDIA_ROOT / "conventions" diff --git a/conventions/views/conventions.py b/conventions/views/conventions.py index 210d38cb9..862c276ca 100644 --- a/conventions/views/conventions.py +++ b/conventions/views/conventions.py @@ -58,7 +58,6 @@ from programmes.models import Financement, NatureLogement from programmes.services import LoyerRedevanceUpdateComputer from upload.models import UploadedFile -from upload.services import UploadService template_sent = "conventions/sent.html" @@ -373,11 +372,16 @@ def get_or_generate_cerfa(request, convention_uuid): if len(files) > 0: file_dict = files[0] uploaded_file = UploadedFile.objects.get(uuid=file_dict["uuid"]) - file = UploadService().get_file(uploaded_file.filepath(convention_uuid)) - return FileResponse( - file, - filename=file_dict["filename"], + default_storage.open( + name=uploaded_file.filepath(convention_uuid), + mode="rb", + ), + filename=( + file_dict["realname"] + if "realname" in file_dict + else file_dict["filename"] + ), as_attachment=True, ) return generate_convention(request, convention_uuid) @@ -662,6 +666,8 @@ def resiliation_start(request, convention_uuid): def display_pdf(request, convention_uuid): # récupérer le doc PDF convention = Convention.objects.get(uuid=convention_uuid) + convention_path = f"conventions/{convention.uuid}/convention_docs" + filename = None if ( convention.statut @@ -673,25 +679,20 @@ def display_pdf(request, convention_uuid): ConventionStatut.A_SIGNER.label, ] and convention.nom_fichier_signe - and default_storage.exists( - f"conventions/{convention.uuid}/convention_docs/{convention.nom_fichier_signe}" - ) + and default_storage.exists(f"{convention_path}/{convention.nom_fichier_signe}") ): filename = convention.nom_fichier_signe - elif default_storage.exists( - f"conventions/{convention.uuid}/convention_docs/{convention.uuid}.pdf" - ): + elif default_storage.exists(f"{convention_path}/{convention.uuid}.pdf"): filename = f"{convention.uuid}.pdf" - elif default_storage.exists( - f"conventions/{convention.uuid}/convention_docs/{convention.uuid}.docx" - ): + elif default_storage.exists(f"{convention_path}/{convention.uuid}.docx"): filename = f"{convention.uuid}.docx" + if filename: return FileResponse( - UploadService( - convention_dirpath=f"conventions/{convention.uuid}/convention_docs", - filename=filename, - ).get_file(), + default_storage.open( + name=f"{convention_path}/{filename}", + mode="rb", + ), filename=filename, ) diff --git a/programmes/fixtures/programmes_for_tests.json b/programmes/fixtures/programmes_for_tests.json index 19d8c61b9..c0b5f9a0d 100644 --- a/programmes/fixtures/programmes_for_tests.json +++ b/programmes/fixtures/programmes_for_tests.json @@ -22,10 +22,10 @@ "nb_locaux_commerciaux": 5, "nb_bureaux": 25, "autres_locaux_hors_convention": "quelques uns", - "reference_publication_acte": "{\"text\": \"this is a test\", \"files\": {\"8c9b6894-1de4-4b45-bd50-e9416dd3ee6b\": {\"uuid\": \"8c9b6894-1de4-4b45-bd50-e9416dd3ee6b\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"0c48af64-c852-4e6f-ae86-265615913ac9\": {\"uuid\": \"0c48af64-c852-4e6f-ae86-265615913ac9\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "reference_publication_acte": "{\"text\": \"this is a test\", \"files\": {\"8c9b6894-1de4-4b45-bd50-e9416dd3ee6b\": {\"uuid\": \"8c9b6894-1de4-4b45-bd50-e9416dd3ee6b\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"0c48af64-c852-4e6f-ae86-265615913ac9\": {\"uuid\": \"0c48af64-c852-4e6f-ae86-265615913ac9\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "effet_relatif": "n'importe quoi", "certificat_adressage": "n'importe quoi", - "reference_cadastrale": "{\"text\": \"this is a test\", \"files\": {\"98ae5093-ff07-4008-bb61-6fa5e7e34fd2\": {\"uuid\": \"98ae5093-ff07-4008-bb61-6fa5e7e34fd2\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"daf5cc14-e5fd-4125-affa-3d7003b1dcc8\": {\"uuid\": \"daf5cc14-e5fd-4125-affa-3d7003b1dcc8\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "reference_cadastrale": "{\"text\": \"this is a test\", \"files\": {\"98ae5093-ff07-4008-bb61-6fa5e7e34fd2\": {\"uuid\": \"98ae5093-ff07-4008-bb61-6fa5e7e34fd2\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"daf5cc14-e5fd-4125-affa-3d7003b1dcc8\": {\"uuid\": \"daf5cc14-e5fd-4125-affa-3d7003b1dcc8\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "permis_construire": "123 456 789 ABC", "date_achevement_previsible": "2024-01-02", "date_achat": "2022-01-02", @@ -58,12 +58,12 @@ "nb_locaux_commerciaux": 5, "nb_bureaux": 25, "autres_locaux_hors_convention": "quelques uns", - "vendeur": "{\"text\": \"this is a test\", \"files\": {\"cff0c56e-40bd-409b-8116-d50430db2239\": {\"uuid\": \"cff0c56e-40bd-409b-8116-d50430db2239\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"a94a4c00-1ee1-4fbd-b200-a758379ec37f\": {\"uuid\": \"a94a4c00-1ee1-4fbd-b200-a758379ec37f\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "vendeur": "{\"text\": \"this is a test\", \"files\": {\"cff0c56e-40bd-409b-8116-d50430db2239\": {\"uuid\": \"cff0c56e-40bd-409b-8116-d50430db2239\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"a94a4c00-1ee1-4fbd-b200-a758379ec37f\": {\"uuid\": \"a94a4c00-1ee1-4fbd-b200-a758379ec37f\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "acquereur": "n'importe quoi", - "reference_notaire": "{\"text\": \"this is a test\", \"files\": {\"45079d28-f737-44b7-9f6a-c1e8367072a5\": {\"uuid\": \"45079d28-f737-44b7-9f6a-c1e8367072a5\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"df747b0d-62c7-4fb3-9fda-37f3b682df80\": {\"uuid\": \"df747b0d-62c7-4fb3-9fda-37f3b682df80\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", - "reference_publication_acte": "{\"text\": \"this is a test\", \"files\": {\"80ecbde5-da41-4ece-a35b-19665b8b2b12\": {\"uuid\": \"80ecbde5-da41-4ece-a35b-19665b8b2b12\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"ae57c1f9-62cb-46df-b496-2f4c78429a4c\": {\"uuid\": \"ae57c1f9-62cb-46df-b496-2f4c78429a4c\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", - "acte_de_propriete": "{\"text\": \"this is a test\", \"files\": {\"8f25d1cd-cb65-4596-9f74-758c6eb00283\": {\"uuid\": \"8f25d1cd-cb65-4596-9f74-758c6eb00283\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"4a5ca2ab-f739-4b9d-ba71-4412268e78f4\": {\"uuid\": \"4a5ca2ab-f739-4b9d-ba71-4412268e78f4\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", - "certificat_adressage": "{\"text\": \"this is a test\", \"files\": {\"4feccfda-205e-42de-b215-46e94b32ddd1\": {\"uuid\": \"4feccfda-205e-42de-b215-46e94b32ddd1\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"6441b179-5ec6-4ca5-bb64-f915f83c34ab\": {\"uuid\": \"6441b179-5ec6-4ca5-bb64-f915f83c34ab\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "reference_notaire": "{\"text\": \"this is a test\", \"files\": {\"45079d28-f737-44b7-9f6a-c1e8367072a5\": {\"uuid\": \"45079d28-f737-44b7-9f6a-c1e8367072a5\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"df747b0d-62c7-4fb3-9fda-37f3b682df80\": {\"uuid\": \"df747b0d-62c7-4fb3-9fda-37f3b682df80\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "reference_publication_acte": "{\"text\": \"this is a test\", \"files\": {\"80ecbde5-da41-4ece-a35b-19665b8b2b12\": {\"uuid\": \"80ecbde5-da41-4ece-a35b-19665b8b2b12\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"ae57c1f9-62cb-46df-b496-2f4c78429a4c\": {\"uuid\": \"ae57c1f9-62cb-46df-b496-2f4c78429a4c\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "acte_de_propriete": "{\"text\": \"this is a test\", \"files\": {\"8f25d1cd-cb65-4596-9f74-758c6eb00283\": {\"uuid\": \"8f25d1cd-cb65-4596-9f74-758c6eb00283\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"4a5ca2ab-f739-4b9d-ba71-4412268e78f4\": {\"uuid\": \"4a5ca2ab-f739-4b9d-ba71-4412268e78f4\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "certificat_adressage": "{\"text\": \"this is a test\", \"files\": {\"4feccfda-205e-42de-b215-46e94b32ddd1\": {\"uuid\": \"4feccfda-205e-42de-b215-46e94b32ddd1\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"6441b179-5ec6-4ca5-bb64-f915f83c34ab\": {\"uuid\": \"6441b179-5ec6-4ca5-bb64-f915f83c34ab\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "permis_construire": "123 456 789 ABC", "date_achevement_previsible": "2024-01-02", "date_achat": "2022-01-02", @@ -81,7 +81,7 @@ "programme": 1, "financement": "PLAI", "type_habitat": "INDIVIDUEL", - "edd_classique": "{\"text\": \"this is a test\", \"files\": {\"fbb9890f-171b-402d-a35e-71e1bd791b70\": {\"uuid\": \"fbb9890f-171b-402d-a35e-71e1bd791b70\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"dccd310d-2e50-45d8-a477-db7b08ae1d71\": {\"uuid\": \"dccd310d-2e50-45d8-a477-db7b08ae1d71\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", + "edd_classique": "{\"text\": \"this is a test\", \"files\": {\"fbb9890f-171b-402d-a35e-71e1bd791b70\": {\"uuid\": \"fbb9890f-171b-402d-a35e-71e1bd791b70\", \"thumbnail\": \"data:image/png;base64,BLAHBLAH==\", \"size\": \"31185\", \"filename\": \"acquereur1.png\", \"realname\": \"acquereur1.png\", \"content_type\": \"image/png\"}, \"dccd310d-2e50-45d8-a477-db7b08ae1d71\": {\"uuid\": \"dccd310d-2e50-45d8-a477-db7b08ae1d71\", \"thumbnail\": \"data:image/png;base64,BLIHBLIH==\", \"size\": \"69076\", \"filename\": \"acquereur2.png\", \"realname\": \"acquereur2.png\", \"content_type\": \"image/png\"}}}", "lgts_mixite_sociale_negocies": 0, "surface_locaux_collectifs_residentiels": 0, "cree_le": "2023-03-08T20:43:46.080Z", diff --git a/templates/common/display_files.html b/templates/common/display_files.html index 49cf10567..722be16e4 100644 --- a/templates/common/display_files.html +++ b/templates/common/display_files.html @@ -4,7 +4,7 @@
- {{file.realname}} + {{file.realname|default:file.filename}} - {{file.size|filesizeformat}} {% endfor %} diff --git a/upload/fixtures/upload_for_tests.json b/upload/fixtures/upload_for_tests.json index af5bf33f1..c65158ccd 100644 --- a/upload/fixtures/upload_for_tests.json +++ b/upload/fixtures/upload_for_tests.json @@ -9,7 +9,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.042Z", - "mis_a_jour_le": "2023-03-08T20:43:46.042Z" + "mis_a_jour_le": "2023-03-08T20:43:46.042Z", + "realname": "acquereur1.png" } }, { @@ -22,7 +23,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.047Z", - "mis_a_jour_le": "2023-03-08T20:43:46.047Z" + "mis_a_jour_le": "2023-03-08T20:43:46.047Z", + "realname": "acquereur2.png" } }, { @@ -35,7 +37,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.048Z", - "mis_a_jour_le": "2023-03-08T20:43:46.048Z" + "mis_a_jour_le": "2023-03-08T20:43:46.048Z", + "realname": "acquereur1.png" } }, { @@ -48,7 +51,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.049Z", - "mis_a_jour_le": "2023-03-08T20:43:46.049Z" + "mis_a_jour_le": "2023-03-08T20:43:46.049Z", + "realname": "acquereur2.png" } }, { @@ -61,7 +65,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.050Z", - "mis_a_jour_le": "2023-03-08T20:43:46.050Z" + "mis_a_jour_le": "2023-03-08T20:43:46.050Z", + "realname": "acquereur1.png" } }, { @@ -74,7 +79,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.051Z", - "mis_a_jour_le": "2023-03-08T20:43:46.051Z" + "mis_a_jour_le": "2023-03-08T20:43:46.051Z", + "realname": "acquereur2.png" } }, { @@ -87,7 +93,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.052Z", - "mis_a_jour_le": "2023-03-08T20:43:46.052Z" + "mis_a_jour_le": "2023-03-08T20:43:46.052Z", + "realname": "acquereur1.png" } }, { @@ -100,7 +107,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.053Z", - "mis_a_jour_le": "2023-03-08T20:43:46.053Z" + "mis_a_jour_le": "2023-03-08T20:43:46.053Z", + "realname": "acquereur2.png" } }, { @@ -113,7 +121,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.054Z", - "mis_a_jour_le": "2023-03-08T20:43:46.054Z" + "mis_a_jour_le": "2023-03-08T20:43:46.054Z", + "realname": "acquereur1.png" } }, { @@ -126,7 +135,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.055Z", - "mis_a_jour_le": "2023-03-08T20:43:46.055Z" + "mis_a_jour_le": "2023-03-08T20:43:46.055Z", + "realname": "acquereur2.png" } }, { @@ -139,7 +149,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.056Z", - "mis_a_jour_le": "2023-03-08T20:43:46.056Z" + "mis_a_jour_le": "2023-03-08T20:43:46.056Z", + "realname": "acquereur1.png" } }, { @@ -152,7 +163,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.058Z", - "mis_a_jour_le": "2023-03-08T20:43:46.058Z" + "mis_a_jour_le": "2023-03-08T20:43:46.058Z", + "realname": "acquereur2.png" } }, { @@ -165,7 +177,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.059Z", - "mis_a_jour_le": "2023-03-08T20:43:46.059Z" + "mis_a_jour_le": "2023-03-08T20:43:46.059Z", + "realname": "acquereur1.png" } }, { @@ -178,7 +191,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.060Z", - "mis_a_jour_le": "2023-03-08T20:43:46.060Z" + "mis_a_jour_le": "2023-03-08T20:43:46.060Z", + "realname": "acquereur2.png" } }, { @@ -191,7 +205,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.061Z", - "mis_a_jour_le": "2023-03-08T20:43:46.061Z" + "mis_a_jour_le": "2023-03-08T20:43:46.061Z", + "realname": "acquereur1.png" } }, { @@ -204,7 +219,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.062Z", - "mis_a_jour_le": "2023-03-08T20:43:46.062Z" + "mis_a_jour_le": "2023-03-08T20:43:46.062Z", + "realname": "acquereur2.png" } }, { @@ -217,7 +233,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.076Z", - "mis_a_jour_le": "2023-03-08T20:43:46.076Z" + "mis_a_jour_le": "2023-03-08T20:43:46.076Z", + "realname": "acquereur1.png" } }, { @@ -230,7 +247,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.077Z", - "mis_a_jour_le": "2023-03-08T20:43:46.077Z" + "mis_a_jour_le": "2023-03-08T20:43:46.077Z", + "realname": "acquereur2.png" } }, { @@ -243,7 +261,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.078Z", - "mis_a_jour_le": "2023-03-08T20:43:46.078Z" + "mis_a_jour_le": "2023-03-08T20:43:46.078Z", + "realname": "acquereur1.png" } }, { @@ -256,7 +275,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.079Z", - "mis_a_jour_le": "2023-03-08T20:43:46.079Z" + "mis_a_jour_le": "2023-03-08T20:43:46.079Z", + "realname": "acquereur2.png" } }, { @@ -269,7 +289,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.090Z", - "mis_a_jour_le": "2023-03-08T20:43:46.090Z" + "mis_a_jour_le": "2023-03-08T20:43:46.090Z", + "realname": "acquereur1.png" } }, { @@ -282,7 +303,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.091Z", - "mis_a_jour_le": "2023-03-08T20:43:46.091Z" + "mis_a_jour_le": "2023-03-08T20:43:46.091Z", + "realname": "acquereur2.png" } }, { @@ -295,7 +317,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.093Z", - "mis_a_jour_le": "2023-03-08T20:43:46.093Z" + "mis_a_jour_le": "2023-03-08T20:43:46.093Z", + "realname": "acquereur1.png" } }, { @@ -308,7 +331,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.094Z", - "mis_a_jour_le": "2023-03-08T20:43:46.094Z" + "mis_a_jour_le": "2023-03-08T20:43:46.094Z", + "realname": "acquereur2.png" } }, { @@ -321,7 +345,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.096Z", - "mis_a_jour_le": "2023-03-08T20:43:46.096Z" + "mis_a_jour_le": "2023-03-08T20:43:46.096Z", + "realname": "acquereur1.png" } }, { @@ -334,7 +359,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.098Z", - "mis_a_jour_le": "2023-03-08T20:43:46.098Z" + "mis_a_jour_le": "2023-03-08T20:43:46.098Z", + "realname": "acquereur2.png" } }, { @@ -347,7 +373,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.113Z", - "mis_a_jour_le": "2023-03-08T20:43:46.113Z" + "mis_a_jour_le": "2023-03-08T20:43:46.113Z", + "realname": "acquereur1.png" } }, { @@ -360,7 +387,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.115Z", - "mis_a_jour_le": "2023-03-08T20:43:46.115Z" + "mis_a_jour_le": "2023-03-08T20:43:46.115Z", + "realname": "acquereur2.png" } }, { @@ -373,7 +401,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.118Z", - "mis_a_jour_le": "2023-03-08T20:43:46.118Z" + "mis_a_jour_le": "2023-03-08T20:43:46.118Z", + "realname": "acquereur1.png" } }, { @@ -386,7 +415,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.118Z", - "mis_a_jour_le": "2023-03-08T20:43:46.118Z" + "mis_a_jour_le": "2023-03-08T20:43:46.118Z", + "realname": "acquereur2.png" } }, { @@ -399,7 +429,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.120Z", - "mis_a_jour_le": "2023-03-08T20:43:46.120Z" + "mis_a_jour_le": "2023-03-08T20:43:46.120Z", + "realname": "acquereur1.png" } }, { @@ -412,7 +443,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.122Z", - "mis_a_jour_le": "2023-03-08T20:43:46.122Z" + "mis_a_jour_le": "2023-03-08T20:43:46.122Z", + "realname": "acquereur2.png" } }, { @@ -425,7 +457,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.124Z", - "mis_a_jour_le": "2023-03-08T20:43:46.124Z" + "mis_a_jour_le": "2023-03-08T20:43:46.124Z", + "realname": "acquereur1.png" } }, { @@ -438,7 +471,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.125Z", - "mis_a_jour_le": "2023-03-08T20:43:46.125Z" + "mis_a_jour_le": "2023-03-08T20:43:46.125Z", + "realname": "acquereur2.png" } }, { @@ -451,7 +485,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.126Z", - "mis_a_jour_le": "2023-03-08T20:43:46.126Z" + "mis_a_jour_le": "2023-03-08T20:43:46.126Z", + "realname": "acquereur1.png" } }, { @@ -464,7 +499,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.127Z", - "mis_a_jour_le": "2023-03-08T20:43:46.127Z" + "mis_a_jour_le": "2023-03-08T20:43:46.127Z", + "realname": "acquereur2.png" } }, { @@ -477,7 +513,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.128Z", - "mis_a_jour_le": "2023-03-08T20:43:46.128Z" + "mis_a_jour_le": "2023-03-08T20:43:46.128Z", + "realname": "acquereur1.png" } }, { @@ -490,7 +527,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.129Z", - "mis_a_jour_le": "2023-03-08T20:43:46.129Z" + "mis_a_jour_le": "2023-03-08T20:43:46.129Z", + "realname": "acquereur2.png" } }, { @@ -503,7 +541,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.130Z", - "mis_a_jour_le": "2023-03-08T20:43:46.130Z" + "mis_a_jour_le": "2023-03-08T20:43:46.130Z", + "realname": "acquereur1.png" } }, { @@ -516,7 +555,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.131Z", - "mis_a_jour_le": "2023-03-08T20:43:46.131Z" + "mis_a_jour_le": "2023-03-08T20:43:46.131Z", + "realname": "acquereur2.png" } }, { @@ -529,7 +569,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.132Z", - "mis_a_jour_le": "2023-03-08T20:43:46.132Z" + "mis_a_jour_le": "2023-03-08T20:43:46.132Z", + "realname": "acquereur1.png" } }, { @@ -542,7 +583,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.133Z", - "mis_a_jour_le": "2023-03-08T20:43:46.133Z" + "mis_a_jour_le": "2023-03-08T20:43:46.133Z", + "realname": "acquereur2.png" } }, { @@ -555,7 +597,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.135Z", - "mis_a_jour_le": "2023-03-08T20:43:46.135Z" + "mis_a_jour_le": "2023-03-08T20:43:46.135Z", + "realname": "acquereur1.png" } }, { @@ -568,7 +611,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.136Z", - "mis_a_jour_le": "2023-03-08T20:43:46.136Z" + "mis_a_jour_le": "2023-03-08T20:43:46.136Z", + "realname": "acquereur2.png" } }, { @@ -581,7 +625,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.139Z", - "mis_a_jour_le": "2023-03-08T20:43:46.139Z" + "mis_a_jour_le": "2023-03-08T20:43:46.139Z", + "realname": "acquereur1.png" } }, { @@ -594,7 +639,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.140Z", - "mis_a_jour_le": "2023-03-08T20:43:46.140Z" + "mis_a_jour_le": "2023-03-08T20:43:46.140Z", + "realname": "acquereur2.png" } }, { @@ -607,7 +653,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.141Z", - "mis_a_jour_le": "2023-03-08T20:43:46.141Z" + "mis_a_jour_le": "2023-03-08T20:43:46.141Z", + "realname": "acquereur1.png" } }, { @@ -620,7 +667,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.143Z", - "mis_a_jour_le": "2023-03-08T20:43:46.143Z" + "mis_a_jour_le": "2023-03-08T20:43:46.143Z", + "realname": "acquereur2.png" } }, { @@ -633,7 +681,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.145Z", - "mis_a_jour_le": "2023-03-08T20:43:46.145Z" + "mis_a_jour_le": "2023-03-08T20:43:46.145Z", + "realname": "acquereur1.png" } }, { @@ -646,7 +695,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.146Z", - "mis_a_jour_le": "2023-03-08T20:43:46.146Z" + "mis_a_jour_le": "2023-03-08T20:43:46.146Z", + "realname": "acquereur2.png" } }, { @@ -659,7 +709,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.148Z", - "mis_a_jour_le": "2023-03-08T20:43:46.148Z" + "mis_a_jour_le": "2023-03-08T20:43:46.148Z", + "realname": "acquereur1.png" } }, { @@ -672,7 +723,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.149Z", - "mis_a_jour_le": "2023-03-08T20:43:46.149Z" + "mis_a_jour_le": "2023-03-08T20:43:46.149Z", + "realname": "acquereur2.png" } }, { @@ -685,7 +737,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.151Z", - "mis_a_jour_le": "2023-03-08T20:43:46.151Z" + "mis_a_jour_le": "2023-03-08T20:43:46.151Z", + "realname": "acquereur1.png" } }, { @@ -698,7 +751,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.152Z", - "mis_a_jour_le": "2023-03-08T20:43:46.152Z" + "mis_a_jour_le": "2023-03-08T20:43:46.152Z", + "realname": "acquereur2.png" } }, { @@ -711,7 +765,8 @@ "size": "31185", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.155Z", - "mis_a_jour_le": "2023-03-08T20:43:46.155Z" + "mis_a_jour_le": "2023-03-08T20:43:46.155Z", + "realname": "acquereur1.png" } }, { @@ -724,7 +779,8 @@ "size": "69076", "content_type": "image/png", "cree_le": "2023-03-08T20:43:46.157Z", - "mis_a_jour_le": "2023-03-08T20:43:46.157Z" + "mis_a_jour_le": "2023-03-08T20:43:46.157Z", + "realname": "acquereur2.png" } } -] +] \ No newline at end of file diff --git a/upload/scripts/__init__.py b/upload/management/commands/__init__.py similarity index 100% rename from upload/scripts/__init__.py rename to upload/management/commands/__init__.py diff --git a/upload/management/commands/fix_text_and_files_fields.py b/upload/management/commands/fix_text_and_files_fields.py new file mode 100644 index 000000000..87a61efba --- /dev/null +++ b/upload/management/commands/fix_text_and_files_fields.py @@ -0,0 +1,108 @@ +import json +import logging +from typing import Any + +from django.core.management.base import BaseCommand + +from conventions.models import Convention +from programmes.models import Lot, Programme + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--dryrun", + action="store_true", + help="Run the command in dry run mode without making any changes", + ) + + def handle(self, *args, **options): + for programme in Programme.objects.all(): + for field in ( + "acquereur", + "acte_de_propriete", + "certificat_adressage", + "edd_stationnements", + "effet_relatif", + "reference_cadastrale", + "reference_notaire", + "reference_publication_acte", + "vendeur", + ): + self._update_instance_field( + instance=programme, field_name=field, dryrun=options["dryrun"] + ) + + for convention in Convention.objects.all(): + for field in ( + "attached", + "commentaires", + "fichier_instruction_denonciation", + "fichier_instruction_resiliation", + "fichier_override_cerfa", + ): + self._update_instance_field( + instance=convention, field_name=field, dryrun=options["dryrun"] + ) + + for lot in Lot.objects.all(): + for field in ( + "edd_classique", + "edd_volumetrique", + ): + self._update_instance_field( + instance=lot, field_name=field, dryrun=options["dryrun"] + ) + + def _update_instance_field( + self, instance: Any, field_name: str, dryrun: bool = False + ): + field = getattr(instance, field_name) + if not field: + return + + self.stdout.write( + f"Processing {instance._meta.object_name} (#{instance.pk}), on field '{field_name}'." + ) + + try: + json_content = json.loads(field) + except json.JSONDecodeError: + self.stdout.write(self.style.ERROR("Failed to decode JSON content")) + return + + if "files" not in json_content: + self.stdout.write( + self.style.ERROR("JSON content does not have a 'files' key") + ) + return + + if not isinstance(json_content["files"], dict): + self.stdout.write( + self.style.ERROR("JSON content 'files' key is not a dict") + ) + return + + is_candidate: bool = False + for v in json_content["files"].values(): + if not is_candidate and "filename" in v: + is_candidate = True + v.update({"realname": v["filename"]}) + + if not is_candidate: + self.stdout.write("No change needed.") + return + + setattr(instance, field_name, json.dumps(json_content)) + + if not dryrun: + instance.save() + self.stdout.write( + self.style.SUCCESS("Done! Updated the instance with the new content.") + ) + else: + self.stdout.write( + self.style.SUCCESS(f">> DRYRUN updated with content: {json_content}") + ) diff --git a/upload/management/commands/fix_upload_files.py b/upload/management/commands/fix_upload_files.py new file mode 100644 index 000000000..716cfc5bf --- /dev/null +++ b/upload/management/commands/fix_upload_files.py @@ -0,0 +1,121 @@ +import json +import re +from collections.abc import Generator +from os.path import splitext +from pathlib import Path + +from django.conf import settings +from django.core.files.storage import default_storage +from django.core.management.base import BaseCommand +from django.db import models, transaction +from django.utils.text import slugify + +from conventions.models import Convention +from upload.models import UploadedFile + + +class Command(BaseCommand): + def handle(self, *args, **options): + for uuids in self._load_uuids_from_log(): + self._handle_conv_upload( + convention_uuid=uuids["convention_uuid"], + upload_file_uuid=uuids["upload_file_uuid"], + ) + + def _load_uuids_from_log(self) -> Generator[dict[str, str]]: + log_filepath = Path( + settings.BASE_DIR, + "upload", + "management", + "commands", + "s3-backup-errors.log", + ) + self.stdout.write(f"Reading log file {log_filepath}") + + pattern = re.compile( + r"apilos-prod/conventions/([0-9a-fA-F-]+)/media/([0-9a-fA-F-]+)" + ) + + with open(log_filepath) as f: + for line in f.readlines(): + parts = pattern.findall(line)[0] + if not len(parts) == 2: + self.stdout.write( + self.style.ERROR( + f"Failed to parse line, expected 2 UUIDs, got {len(parts)}" + ) + ) + continue + + yield { + "convention_uuid": parts[0], + "upload_file_uuid": parts[1], + } + + def _get_conv_upload_fields(self) -> list[str]: + return [ + field.name + for field in Convention._meta.get_fields() + if isinstance(field, models.TextField) + ] + + def _handle_conv_upload(self, convention_uuid: str, upload_file_uuid: str): + convention = Convention.objects.filter(uuid=convention_uuid).first() + if convention is None: + self.stdout.write( + self.style.ERROR(f"Convention {convention_uuid} not found") + ) + return + + uploaded_file = UploadedFile.objects.filter(uuid=upload_file_uuid).first() + if uploaded_file is None: + self.stdout.write( + self.style.ERROR(f"UploadedFile {upload_file_uuid} not found") + ) + return + + self.stdout.write( + f"Processing convention {convention_uuid} and upload {upload_file_uuid}" + ) + + for field_name in self._get_conv_upload_fields(): + if not (field_value := getattr(convention, field_name)): + continue + try: + json_field_value = json.loads(field_value) + except json.JSONDecodeError: + continue + if upload_file_uuid not in json_field_value["files"]: + continue + + # get the current upload data + upload_filepath = uploaded_file.filepath(convention_uuid) + upload_content = default_storage.open( + uploaded_file.filepath(convention_uuid), "rb" + ) + + # compute the new file name + name, extension = splitext( + json_field_value["files"][upload_file_uuid]["filename"] + ) + new_filename = f"{slugify(name)}{extension}" + self.stdout.write( + f"Changing {json_field_value['files'][upload_file_uuid]['filename']} to {new_filename}" + ) + + with transaction.atomic(): + # update the convention field value + json_field_value["files"][upload_file_uuid]["filename"] = new_filename + setattr(convention, field_name, json.dumps(json_field_value)) + convention.save() + + # update the upload file name + uploaded_file.filename = new_filename + uploaded_file.save() + + # save the new file on S3 + new_upload_filepath = uploaded_file.filepath(convention_uuid) + default_storage.save(name=new_upload_filepath, content=upload_content) + + # delete the previous file on S3 + default_storage.delete(upload_filepath) diff --git a/upload/scripts/s3-backup-errors.log b/upload/management/commands/s3-backup-errors.log similarity index 100% rename from upload/scripts/s3-backup-errors.log rename to upload/management/commands/s3-backup-errors.log diff --git a/upload/migrations/0009_migrate_text_and_files_fields.py b/upload/migrations/0009_migrate_text_and_files_fields.py deleted file mode 100644 index 4e565b62a..000000000 --- a/upload/migrations/0009_migrate_text_and_files_fields.py +++ /dev/null @@ -1,91 +0,0 @@ -# Generated by Django 5.1 on 2024-09-11 08:35 - -import json -import logging -from typing import Any - -from django.db import migrations - -logger = logging.getLogger(__name__) - - -def _update_instance_field(instance: Any, field_name: str): - field = getattr(instance, field_name) - if not field: - return - - try: - json_content = json.loads(field) - except json.JSONDecodeError: - logger.error(f"Failed to decode JSON content for {instance} field {field_name}") - return - - if "files" not in json_content: - logger.error( - f"JSON content for {instance} field {field_name} does not contain 'files' key" - ) - return - if not isinstance(json_content["files"], dict): - logger.error( - f"JSON content for {instance} field {field_name} 'files' key is not a dict" - ) - return - - for v in json_content["files"].values(): - v.update({"realname": v["filename"]}) - - setattr(instance, field_name, json.dumps(json_content)) - instance.save() - - -def migrate_text_and_files_fields(apps, schema_editor): - Convention = apps.get_model("conventions", "Convention") - Lot = apps.get_model("programmes", "Lot") - Programme = apps.get_model("programmes", "Programme") - - for programme in Programme.objects.all(): - for field in ( - "acquereur", - "acte_de_propriete", - "certificat_adressage", - "edd_stationnements", - "effet_relatif", - "reference_cadastrale", - "reference_notaire", - "reference_publication_acte", - "vendeur", - ): - _update_instance_field(instance=programme, field_name=field) - - for convention in Convention.objects.all(): - for field in ( - "attached", - "commentaires", - "fichier_instruction_denonciation", - "fichier_instruction_resiliation", - "fichier_override_cerfa", - ): - _update_instance_field(instance=convention, field_name=field) - - for lot in Lot.objects.all(): - for field in ( - "edd_classique", - "edd_volumetrique", - ): - _update_instance_field(instance=lot, field_name=field) - - # FIXME: for dev purpose only - assert False - - -class Migration(migrations.Migration): - dependencies = [ - ("upload", "0008_alter_uploadedfile_filename_and_more"), - ] - - operations = [ - migrations.RunPython( - code=migrate_text_and_files_fields, - reverse_code=migrations.RunPython.noop, - ), - ] diff --git a/upload/scripts/fix_upload_files.py b/upload/scripts/fix_upload_files.py deleted file mode 100644 index 87bef25b2..000000000 --- a/upload/scripts/fix_upload_files.py +++ /dev/null @@ -1,101 +0,0 @@ -# python manage.py shell < upload/scripts/fix_upload_files.py - -import json -import re -from os.path import splitext -from pathlib import Path - -from django.conf import settings -from django.core.files.storage import default_storage -from django.db import models, transaction -from django.utils.text import slugify - -from conventions.models import Convention -from upload.models import UploadedFile - - -def _load_uuids_from_log() -> list[dict[str, str]]: - uuids = [] - - log_filepath = Path(settings.BASE_DIR, "upload", "scripts", "s3-backup-errors.log") - with open(log_filepath) as f: - pattern = re.compile( - r"apilos-prod/conventions/([0-9a-fA-F-]+)/media/([0-9a-fA-F-]+)" - ) - for line in f.readlines(): - parts = pattern.findall(line)[0] - assert len(parts) == 2, f"Expected 2 UUIDs, got {len(parts)}" - uuids.append( - { - "convention_uuid": parts[0], - "upload_file_uuid": parts[1], - } - ) - - return uuids - - -def _get_conv_upload_fields() -> list[str]: - return [ - field.name - for field in Convention._meta.get_fields() - if isinstance(field, models.TextField) - ] - - -def handle_conv_upload(convention_uuid: str, upload_file_uuid: str): - convention = Convention.objects.filter(uuid=convention_uuid).first() - assert convention is not None, f"Convention {convention_uuid} not found" - - uploaded_file = UploadedFile.objects.filter(uuid=upload_file_uuid).first() - assert uploaded_file is not None, f"UploadedFile {upload_file_uuid} not found" - - for field_name in _get_conv_upload_fields(): - if not (field_value := getattr(convention, field_name)): - continue - try: - json_field_value = json.loads(field_value) - except json.JSONDecodeError: - continue - if upload_file_uuid not in json_field_value["files"]: - continue - - # get the current upload data - upload_filepath = uploaded_file.filepath(convention_uuid) - upload_content = default_storage.open( - uploaded_file.filepath(convention_uuid), "rb" - ) - - # compute the new file name - name, extension = splitext( - json_field_value["files"][upload_file_uuid]["filename"] - ) - new_filename = f"{slugify(name)}{extension}" - - with transaction.atomic(): - # update the convention field value - json_field_value["files"][upload_file_uuid]["filename"] = new_filename - setattr(convention, field_name, json.dumps(json_field_value)) - convention.save() - - # update the upload file name - uploaded_file.filename = new_filename - uploaded_file.save() - - # save the new file on S3 - new_upload_filepath = uploaded_file.filepath(convention_uuid) - default_storage.save(name=new_upload_filepath, content=upload_content) - - # delete the previous file on S3 - default_storage.delete(upload_filepath) - - -def main(): - for uuids in _load_uuids_from_log(): - handle_conv_upload( - convention_uuid=uuids["convention_uuid"], - upload_file_uuid=uuids["upload_file_uuid"], - ) - - -main() diff --git a/upload/services.py b/upload/services.py index 2a244e12d..3874a0267 100644 --- a/upload/services.py +++ b/upload/services.py @@ -56,13 +56,6 @@ def upload_file_io(self, file_io) -> None: destination.write(file_io.getbuffer()) destination.close() - def get_file(self, filepath=None) -> File: - filepath = filepath or self.path - return default_storage.open( - filepath, - "rb", - ) - @property def path(self): return f"{self.convention_dirpath}/{self.filename}" diff --git a/upload/tests/test_services.py b/upload/tests/test_services.py index 39c9e0ec1..4dcba98f4 100644 --- a/upload/tests/test_services.py +++ b/upload/tests/test_services.py @@ -2,7 +2,6 @@ from pathlib import Path from django.conf import settings -from django.core.files.storage import default_storage from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase @@ -47,24 +46,3 @@ def test_upload_file_io(self): def test_path(self): self.assertEqual(self.upload_service.path, "some/path/a_file.pdf") - - def test_get_file_without_parameter(self): - sample_uploaded_file = SimpleUploadedFile( - self.some_filename, b"file_content", content_type="application/pdf" - ) - self.upload_service.upload_file(sample_uploaded_file) - self.assertEqual( - self.upload_service.get_file().read(), - default_storage.open(self.upload_service.path).read(), - ) - - def test_get_file_with_parameter(self): - parent_filepath = Path(settings.MEDIA_ROOT, "another/path") - parent_filepath.mkdir(parents=True, exist_ok=True) - another_filepath = parent_filepath / self.some_filename - another_filepath.write_text("Comment est votre blanquette ?") - - self.assertEqual( - self.upload_service.get_file(another_filepath).read(), - b"Comment est votre blanquette ?", - ) diff --git a/upload/views.py b/upload/views.py index f197c3b10..4e49c0601 100644 --- a/upload/views.py +++ b/upload/views.py @@ -1,3 +1,4 @@ +from django.core.files.storage import default_storage from django.http.response import FileResponse, JsonResponse from django.views.decorators.http import require_GET, require_POST @@ -31,11 +32,12 @@ def _compute_dirpath(request): @require_GET def display_file(request, convention_uuid, uploaded_file_uuid): uploaded_file = UploadedFile.objects.get(uuid=uploaded_file_uuid) - file = UploadService().get_file(uploaded_file.filepath(convention_uuid)) - return FileResponse( - file, - filename=uploaded_file.filename, + default_storage.open( + name=uploaded_file.filepath(convention_uuid), + mode="rb", + ), + filename=uploaded_file.realname, as_attachment=True, )