From e5041b5b9283a4cfd6c25b8512cd2ed46cb7f091 Mon Sep 17 00:00:00 2001 From: "Nico Pfiffner (nipfif)" Date: Sun, 3 Dec 2023 14:53:42 +0100 Subject: [PATCH] feat: Add option to enable/disable regex matching for expected fields in file blueprint settings. --- ddm/forms.py | 5 +- ...lueprint_expected_fields_regex_matching.py | 18 +++++++ ddm/models/core.py | 7 +++ ddm/static/ddm/vue/js/vue_questionnaire.js | 2 +- ddm/static/ddm/vue/js/vue_uploader.js | 6 +-- ddm/static/ddm/vue/webpack-stats.json | 4 +- .../block_data_extraction.html | 15 ++++++ .../block_file_validation.html | 17 ++++++ .../donation_blueprint/create.html | 53 ++----------------- .../donation_blueprint/edit.html | 41 ++------------ docs/modules/ROOT/pages/for_researchers.adoc | 7 ++- frontend/src/components/FileUploader.vue | 23 +++++--- 12 files changed, 96 insertions(+), 102 deletions(-) create mode 100644 ddm/migrations/0046_donationblueprint_expected_fields_regex_matching.py create mode 100644 ddm/templates/ddm/admin/data_donation/donation_blueprint/block_data_extraction.html create mode 100644 ddm/templates/ddm/admin/data_donation/donation_blueprint/block_file_validation.html diff --git a/ddm/forms.py b/ddm/forms.py index 29fd2465..d76609e4 100644 --- a/ddm/forms.py +++ b/ddm/forms.py @@ -82,8 +82,9 @@ class BlueprintEditForm(forms.ModelForm): class Meta: model = DonationBlueprint - fields = ['name', 'description', 'regex_path', 'exp_file_format', 'csv_delimiter', - 'file_uploader', 'json_extraction_root', 'expected_fields'] + fields = ['name', 'description', 'regex_path', 'exp_file_format', + 'csv_delimiter', 'file_uploader', 'json_extraction_root', + 'expected_fields', 'expected_fields_regex_matching'] widgets = { 'expected_fields': forms.Textarea(attrs={'rows': 1}), 'regex_path': forms.Textarea(attrs={'rows': 1}), diff --git a/ddm/migrations/0046_donationblueprint_expected_fields_regex_matching.py b/ddm/migrations/0046_donationblueprint_expected_fields_regex_matching.py new file mode 100644 index 00000000..8adf024c --- /dev/null +++ b/ddm/migrations/0046_donationblueprint_expected_fields_regex_matching.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.13 on 2023-12-03 13:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ddm', '0045_alter_processingrule_comparison_operator'), + ] + + operations = [ + migrations.AddField( + model_name='donationblueprint', + name='expected_fields_regex_matching', + field=models.BooleanField(default=False, help_text='Select if you use regex expressions in the "Excpected fields".'), + ), + ] diff --git a/ddm/models/core.py b/ddm/models/core.py index 180e0293..20db8f09 100644 --- a/ddm/models/core.py +++ b/ddm/models/core.py @@ -466,6 +466,12 @@ class FileFormats(models.TextChoices): ) ) + expected_fields_regex_matching = models.BooleanField( + default=False, + null=False, + help_text='Select if you use regex expressions in the "Excpected fields".' + ) + file_uploader = models.ForeignKey( 'FileUploader', null=True, @@ -505,6 +511,7 @@ def get_config(self): 'format': self.exp_file_format, 'json_extraction_root': self.json_extraction_root, 'expected_fields': json.loads("[" + str(self.expected_fields) + "]"), + 'exp_fields_regex_matching': self.expected_fields_regex_matching, 'fields_to_extract': self.get_fields_to_extract(), 'regex_path': self.regex_path, 'filter_rules': self.get_filter_rules(), diff --git a/ddm/static/ddm/vue/js/vue_questionnaire.js b/ddm/static/ddm/vue/js/vue_questionnaire.js index 03841f04..6eb08c09 100644 --- a/ddm/static/ddm/vue/js/vue_questionnaire.js +++ b/ddm/static/ddm/vue/js/vue_questionnaire.js @@ -590,7 +590,7 @@ eval("// style-loader: Adds some css to the DOM by adding a