From 350f0726868cbd29b9debc1e8b81c504348c7e93 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 27 Jun 2023 16:06:39 -0700 Subject: [PATCH 1/2] Updating DSR data pre-processor to support manual data --- .../privacy_request/dsr_package/dsr_report_builder.py | 5 ++++- tests/ops/service/test_storage_uploader_service.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py b/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py index 5fd2d12a6a..97ef6d5d0f 100644 --- a/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py +++ b/src/fides/api/service/privacy_request/dsr_package/dsr_report_builder.py @@ -155,7 +155,10 @@ def generate(self) -> BytesIO: # pre-process data to split the dataset:collection keys datasets: Dict[str, Any] = defaultdict(lambda: defaultdict(list)) for key, rows in self.dsr_data.items(): - [dataset_name, collection_name] = key.split(":") + parts = key.split(":", 1) + dataset_name, collection_name = ( + parts if len(parts) > 1 else ("manual", parts[0]) + ) datasets[dataset_name][collection_name].extend(rows) for dataset_name, collections in datasets.items(): diff --git a/tests/ops/service/test_storage_uploader_service.py b/tests/ops/service/test_storage_uploader_service.py index 82298a4bf6..9be73786e5 100644 --- a/tests/ops/service/test_storage_uploader_service.py +++ b/tests/ops/service/test_storage_uploader_service.py @@ -354,6 +354,7 @@ def data(self) -> Dict[str, Any]: {"uuid": "xyz-122-333", "name": "foo1", "email": "foo@bar1"}, ], "mongo:foobar": [{"_id": 1, "customer": {"x": 1, "y": [1, 2]}}], + "filing_cabinet": [{"id": "123"}], # represents manual data } def test_json_data(self, data, privacy_request): @@ -367,6 +368,7 @@ def test_csv_format(self, data, privacy_request): zipfile = ZipFile(buff) assert zipfile.namelist() == [ + "filing_cabinet.csv", "mongo:address.csv", "mysql:customer.csv", "mongo:foobar.csv", @@ -435,6 +437,9 @@ def test_html_format(self, data, privacy_request): assert zipfile.namelist() == [ "/main.css", "/back.svg", + "/manual/index.html", + "/manual/filing_cabinet/1.html", + "/manual/filing_cabinet/index.html", "/mongo/address/1.html", "/mongo/address/2.html", "/mongo/address/index.html", From 89619f75fdf94420f38af235190afb42b25644b3 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 28 Jun 2023 15:20:21 -0700 Subject: [PATCH 2/2] Fixing tests --- tests/ops/service/test_storage_uploader_service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ops/service/test_storage_uploader_service.py b/tests/ops/service/test_storage_uploader_service.py index 9be73786e5..22e9366109 100644 --- a/tests/ops/service/test_storage_uploader_service.py +++ b/tests/ops/service/test_storage_uploader_service.py @@ -368,10 +368,10 @@ def test_csv_format(self, data, privacy_request): zipfile = ZipFile(buff) assert zipfile.namelist() == [ - "filing_cabinet.csv", "mongo:address.csv", "mysql:customer.csv", "mongo:foobar.csv", + "filing_cabinet.csv", ] with zipfile.open("mongo:address.csv") as address_csv: @@ -437,9 +437,6 @@ def test_html_format(self, data, privacy_request): assert zipfile.namelist() == [ "/main.css", "/back.svg", - "/manual/index.html", - "/manual/filing_cabinet/1.html", - "/manual/filing_cabinet/index.html", "/mongo/address/1.html", "/mongo/address/2.html", "/mongo/address/index.html", @@ -450,6 +447,9 @@ def test_html_format(self, data, privacy_request): "/mysql/customer/2.html", "/mysql/customer/index.html", "/mysql/index.html", + "/manual/filing_cabinet/1.html", + "/manual/filing_cabinet/index.html", + "/manual/index.html", "/index.html", ]