From 10a927f9039afeda62566ddd4bb74672c34ebfd9 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 19 Dec 2024 09:49:40 +0100 Subject: [PATCH 01/15] unique unicef_id constraint --- .../household/migrations/0005_migration.py | 21 +++++++++++++++++++ src/hct_mis_api/apps/household/models.py | 14 +++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 src/hct_mis_api/apps/household/migrations/0005_migration.py diff --git a/src/hct_mis_api/apps/household/migrations/0005_migration.py b/src/hct_mis_api/apps/household/migrations/0005_migration.py new file mode 100644 index 0000000000..ad31fcdc79 --- /dev/null +++ b/src/hct_mis_api/apps/household/migrations/0005_migration.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.25 on 2024-12-19 08:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('household', '0004_migration'), + ] + + operations = [ + migrations.AddConstraint( + model_name='household', + constraint=models.UniqueConstraint(condition=models.Q(('is_removed', False)), fields=('unicef_id', 'program'), name='unique_ind_unicef_id_in_program'), + ), + migrations.AddConstraint( + model_name='individual', + constraint=models.UniqueConstraint(condition=models.Q(('is_removed', False), ('duplicate', False)), fields=('unicef_id', 'program'), name='unique_hh_unicef_id_in_program'), + ), + ] diff --git a/src/hct_mis_api/apps/household/models.py b/src/hct_mis_api/apps/household/models.py index ca48ab5733..39040bd2ed 100644 --- a/src/hct_mis_api/apps/household/models.py +++ b/src/hct_mis_api/apps/household/models.py @@ -576,6 +576,13 @@ class CollectType(models.TextChoices): class Meta: verbose_name = "Household" permissions = (("can_withdrawn", "Can withdrawn Household"),) + constraints = [ + UniqueConstraint( + fields=["unicef_id", "program"], + condition=Q(is_removed=False), + name="unique_ind_unicef_id_in_program", + ) + ] def save(self, *args: Any, **kwargs: Any) -> None: from hct_mis_api.apps.targeting.models import ( @@ -1187,6 +1194,13 @@ def __str__(self) -> str: class Meta: verbose_name = "Individual" indexes = (GinIndex(fields=["vector_column"]),) + constraints = [ + UniqueConstraint( + fields=["unicef_id", "program"], + condition=Q(is_removed=False) & Q(duplicate=False), + name="unique_hh_unicef_id_in_program", + ) + ] def recalculate_data(self, save: bool = True) -> Tuple[Any, List[str]]: update_fields = ["disability"] From ef7080299b1bfa482c4e38b9b4f23bda68c89d68 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 19 Dec 2024 10:35:12 +0100 Subject: [PATCH 02/15] fix test_double_entries --- src/hct_mis_api/api/endpoints/rdi/upload.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hct_mis_api/api/endpoints/rdi/upload.py b/src/hct_mis_api/api/endpoints/rdi/upload.py index 39fc43705f..1eb1c84910 100644 --- a/src/hct_mis_api/api/endpoints/rdi/upload.py +++ b/src/hct_mis_api/api/endpoints/rdi/upload.py @@ -140,6 +140,7 @@ class Meta: "updated_at", "version", "vector_column", + "unicef_id", ] def validate_role(self, value: str) -> Optional[str]: @@ -175,6 +176,7 @@ class Meta: "geopoint", "detail_id", "version", + "unicef_id", ] validators = [HouseholdValidator()] From c17c42b6c95179b70a7d8f784af651e7e6005b59 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 19 Dec 2024 10:52:13 +0100 Subject: [PATCH 03/15] fix household test_models --- tests/unit/apps/household/test_models.py | 27 +++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/tests/unit/apps/household/test_models.py b/tests/unit/apps/household/test_models.py index d4522a1ed0..8322dfb3e1 100644 --- a/tests/unit/apps/household/test_models.py +++ b/tests/unit/apps/household/test_models.py @@ -120,9 +120,9 @@ class TestDocument(TestCase): def setUpTestData(cls) -> None: super().setUpTestData() call_command("loadcountries") - business_area = create_afghanistan() + cls.business_area = create_afghanistan() afghanistan = Country.objects.get(name="Afghanistan") - _, (individual,) = create_household(household_args={"size": 1, "business_area": business_area}) + _, (individual,) = create_household(household_args={"size": 1, "business_area": cls.business_area}) cls.country = afghanistan cls.individual = individual @@ -187,10 +187,9 @@ def test_create_representation_with_the_same_number(self) -> None: program_3 = ProgramFactory() program_4 = ProgramFactory() - for _program in [program_1, program_2]: - (individual_to_create, documents_to_create, _, _) = copy_individual_fast(self.individual, _program) - Individual.objects.bulk_create([individual_to_create]) - Document.objects.bulk_create(documents_to_create) + (individual_to_create, documents_to_create, _, _) = copy_individual_fast(self.individual, program_2) + Individual.objects.bulk_create([individual_to_create]) + Document.objects.bulk_create(documents_to_create) # test regular create for _program in [program_3, program_4]: @@ -209,8 +208,7 @@ def test_create_representation_with_the_same_number(self) -> None: ) # don't allow to create representations with the same document number and programs - (individual_to_create, _, _, _) = copy_individual_fast(self.individual, _program) - (created_individual_representation,) = Individual.objects.bulk_create([individual_to_create]) + _, (individual,) = create_household(household_args={"size": 1, "business_area": self.business_area, "program": program_1}) with self.assertRaises(IntegrityError): with transaction.atomic(): # bulk create @@ -218,7 +216,7 @@ def test_create_representation_with_the_same_number(self) -> None: [ Document( document_number="213123", - individual=created_individual_representation, + individual=individual, country=self.country, type=document_type, status=Document.STATUS_VALID, @@ -234,7 +232,7 @@ def test_create_representation_with_the_same_number(self) -> None: # regular create Document.objects.create( document_number="213123", - individual=created_individual_representation, + individual=individual, country=self.country, type=document_type, status=Document.STATUS_VALID, @@ -426,11 +424,10 @@ def test_create_representations_duplicated_documents_with_different_numbers_and_ program_2 = ProgramFactory() program_3 = ProgramFactory() - # make representations with the same number - for _program in [program_1, program_2]: - (individual_to_create, documents_to_create, _, _) = copy_individual_fast(self.individual, _program) - Individual.objects.bulk_create([individual_to_create]) - Document.objects.bulk_create(documents_to_create) + # make representation with the same number + (individual_to_create, documents_to_create, _, _) = copy_individual_fast(self.individual, program_2) + Individual.objects.bulk_create([individual_to_create]) + Document.objects.bulk_create(documents_to_create) # make representation with different number program_3_individual_representation = (individual_to_create, _, _, _) = copy_individual_fast( From e4448715793b50a965932b4e5fde933915120f7f Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 19 Dec 2024 11:33:25 +0100 Subject: [PATCH 04/15] change exclusion for importing hh and inds to another program --- src/hct_mis_api/apps/registration_datahub/mutations.py | 10 ++++++++-- .../tasks/import_program_population.py | 10 ++++++++-- .../test_program_population_to_pending_objects.py | 2 ++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/hct_mis_api/apps/registration_datahub/mutations.py b/src/hct_mis_api/apps/registration_datahub/mutations.py index cd2147b801..e9e0befa78 100644 --- a/src/hct_mis_api/apps/registration_datahub/mutations.py +++ b/src/hct_mis_api/apps/registration_datahub/mutations.py @@ -115,15 +115,21 @@ def create_registration_data_import_for_import_program_population( pull_pictures = registration_data_import_data.pop("pull_pictures", True) screen_beneficiary = registration_data_import_data.pop("screen_beneficiary", False) import_from_program_id = registration_data_import_data.pop("import_from_program_id", None) + households_to_exclude = Household.all_merge_status_objects.filter( + program=import_to_program_id, + ).values_list("unicef_id", flat=True) households = Household.objects.filter( program_id=import_from_program_id, withdrawn=False, - ).exclude(household_collection__households__program=import_to_program_id) + ).exclude(unicef_id__in=households_to_exclude) + individuals_to_exclude = Individual.all_merge_status_objects.filter( + program=import_to_program_id, + ).values_list("unicef_id", flat=True) individuals = Individual.objects.filter( program_id=import_from_program_id, withdrawn=False, duplicate=False, - ).exclude(individual_collection__individuals__program=import_to_program_id) + ).exclude(unicef_id__in=individuals_to_exclude) created_obj_hct = RegistrationDataImport( status=RegistrationDataImport.IMPORTING, imported_by=user, diff --git a/src/hct_mis_api/apps/registration_datahub/tasks/import_program_population.py b/src/hct_mis_api/apps/registration_datahub/tasks/import_program_population.py index 95b6ad9cac..0dfad51a54 100644 --- a/src/hct_mis_api/apps/registration_datahub/tasks/import_program_population.py +++ b/src/hct_mis_api/apps/registration_datahub/tasks/import_program_population.py @@ -8,17 +8,23 @@ def import_program_population( import_from_program_id: str, import_to_program_id: str, rdi: RegistrationDataImport ) -> None: + households_to_exclude = Household.all_merge_status_objects.filter( + program=import_to_program_id, + ).values_list("unicef_id", flat=True) copy_from_households = Household.objects.filter( program=import_from_program_id, withdrawn=False, - ).exclude(household_collection__households__program_id=import_to_program_id) + ).exclude(unicef_id__in=households_to_exclude) + individuals_to_exclude = Individual.all_merge_status_objects.filter( + program=import_to_program_id, + ).values_list("unicef_id", flat=True) copy_from_individuals = ( Individual.objects.filter( program_id=import_from_program_id, withdrawn=False, duplicate=False, ) - .exclude(individual_collection__individuals__program_id=import_to_program_id) + .exclude(unicef_id__in=individuals_to_exclude) .order_by("first_registration_date") ) import_to_program = Program.objects.get(id=import_to_program_id) diff --git a/tests/unit/apps/registration_datahub/test_program_population_to_pending_objects.py b/tests/unit/apps/registration_datahub/test_program_population_to_pending_objects.py index f57585a4a3..1b4fdb6761 100644 --- a/tests/unit/apps/registration_datahub/test_program_population_to_pending_objects.py +++ b/tests/unit/apps/registration_datahub/test_program_population_to_pending_objects.py @@ -520,10 +520,12 @@ def test_not_import_excluded_objects(self) -> None: household_already_in_program.household_collection = household_collection household_already_in_program.save() household_already_in_program_repr.household_collection = household_collection + household_already_in_program_repr.unicef_id = household_already_in_program.unicef_id household_already_in_program_repr.save() individuals_already_in_program[0].individual_collection = individual_collection individuals_already_in_program[0].save() individuals_already_in_program_repr[0].individual_collection = individual_collection + individuals_already_in_program_repr[0].unicef_id = individuals_already_in_program[0].unicef_id individuals_already_in_program_repr[0].save() self._object_count_before_after() From 40e76e78701de1f1b5cc2b6b5cc206e39f0c83c2 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 19 Dec 2024 11:41:04 +0100 Subject: [PATCH 05/15] fix test_create_pending_objects_from_objects --- ...t_program_population_to_pending_objects.py | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/tests/unit/apps/registration_datahub/test_program_population_to_pending_objects.py b/tests/unit/apps/registration_datahub/test_program_population_to_pending_objects.py index f57585a4a3..9826264455 100644 --- a/tests/unit/apps/registration_datahub/test_program_population_to_pending_objects.py +++ b/tests/unit/apps/registration_datahub/test_program_population_to_pending_objects.py @@ -456,32 +456,27 @@ def test_create_pending_objects_from_objects(self) -> None: pending_individual_role_in_household.individual, pending_individuals.exclude(relationship=HEAD).first(), ) - for _ in range(10): - registration_data_import = RegistrationDataImportFactory( - business_area=self.afghanistan, - program=self.program_to, - ) - import_program_population( - import_from_program_id=str(self.program_from.id), - import_to_program_id=str(self.program_to.id), - rdi=registration_data_import, - ) - pending_household = Household.pending_objects.order_by("created_at").last() - pending_individual1 = Individual.pending_objects.order_by("-created_at")[0] - pending_individual2 = Individual.pending_objects.order_by("-created_at")[1] - - self.assertIn( - pending_household.head_of_household, - [pending_individual1, pending_individual2], - ) - self.assertEqual( - pending_individual1.household, - pending_household, - ) - self.assertEqual( - pending_individual2.household, - pending_household, - ) + registration_data_import = RegistrationDataImportFactory( + business_area=self.afghanistan, + program=self.program_to, + ) + import_program_population( + import_from_program_id=str(self.program_from.id), + import_to_program_id=str(self.program_to.id), + rdi=registration_data_import, + ) + pending_household_count = ( + Household.pending_objects.filter(registration_data_import=registration_data_import) + .order_by("created_at") + .count() + ) + pending_individual_count = ( + Individual.pending_objects.filter(registration_data_import=registration_data_import) + .order_by("-created_at") + .count() + ) + self.assertEqual(pending_household_count, 0) + self.assertEqual(pending_individual_count, 0) def test_not_import_excluded_objects(self) -> None: household_withdrawn, individuals = create_household_and_individuals( From 86a6f03efef217296b7ec53dd30aa1b3e17826e5 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 19 Dec 2024 11:44:54 +0100 Subject: [PATCH 06/15] fix linters --- tests/unit/apps/household/test_models.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/unit/apps/household/test_models.py b/tests/unit/apps/household/test_models.py index 8322dfb3e1..ad81fb4b02 100644 --- a/tests/unit/apps/household/test_models.py +++ b/tests/unit/apps/household/test_models.py @@ -208,7 +208,9 @@ def test_create_representation_with_the_same_number(self) -> None: ) # don't allow to create representations with the same document number and programs - _, (individual,) = create_household(household_args={"size": 1, "business_area": self.business_area, "program": program_1}) + _, (individual,) = create_household( + household_args={"size": 1, "business_area": self.business_area, "program": program_1} + ) with self.assertRaises(IntegrityError): with transaction.atomic(): # bulk create @@ -419,8 +421,6 @@ def test_create_representations_duplicated_documents_with_different_numbers_and_ # allow to create representations with the same document number within different programs self.individual.is_original = True self.individual.save() - - program_1 = self.individual.program program_2 = ProgramFactory() program_3 = ProgramFactory() @@ -430,9 +430,6 @@ def test_create_representations_duplicated_documents_with_different_numbers_and_ Document.objects.bulk_create(documents_to_create) # make representation with different number - program_3_individual_representation = (individual_to_create, _, _, _) = copy_individual_fast( - self.individual, program_3 - ) (program_3_individual_representation,) = Individual.objects.bulk_create([individual_to_create]) Document.objects.create( document_number="456", From a3af87c7f85da924845f1f5ef562714b27058718 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 19 Dec 2024 11:50:03 +0100 Subject: [PATCH 07/15] fix test_create_targeting_for_people --- tests/selenium/targeting/test_targeting.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/selenium/targeting/test_targeting.py b/tests/selenium/targeting/test_targeting.py index a4ef66b0e1..3ecba9e3ea 100644 --- a/tests/selenium/targeting/test_targeting.py +++ b/tests/selenium/targeting/test_targeting.py @@ -168,7 +168,6 @@ def create_custom_household(observed_disability: list[str], residence_status: st program = Program.objects.get(name="Test Programm") household, _ = create_household_and_individuals( household_data={ - "unicef_id": "HH-00-0000.0442", "rdi_merge_status": "MERGED", "business_area": program.business_area, "program": program, @@ -235,7 +234,6 @@ def create_targeting(household_without_disabilities: Household) -> TargetPopulat target_population.save() household, _ = create_household( household_args={ - "unicef_id": "HH-00-0000.0442", "business_area": program.business_area, "program": program, "residence_status": HOST, From 48c162793cb5e5e0fc8386b29e250f7657ba2264 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 19 Dec 2024 12:01:09 +0100 Subject: [PATCH 08/15] add models for new constraints --- pyproject.toml | 2 +- src/frontend/package.json | 2 +- tests/unit/apps/household/test_models.py | 24 +++++++++++++++++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e145b040bb..6b9c284d5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,7 +137,7 @@ distribution = true [project] name = "hope" -version = "2.15.0" +version = "2.16.0" description = "HCT MIS is UNICEF's humanitarian cash transfer platform." authors = [ { name = "Tivix" }, diff --git a/src/frontend/package.json b/src/frontend/package.json index 8b0ca87c11..71f65a10ae 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -1,6 +1,6 @@ { "name": "frontend", - "version": "2.15.0", + "version": "2.16.0", "private": true, "type": "module", "scripts": { diff --git a/tests/unit/apps/household/test_models.py b/tests/unit/apps/household/test_models.py index ad81fb4b02..b8b81a3073 100644 --- a/tests/unit/apps/household/test_models.py +++ b/tests/unit/apps/household/test_models.py @@ -7,7 +7,12 @@ from hct_mis_api.apps.core.utils import IDENTIFICATION_TYPE_TO_KEY_MAPPING from hct_mis_api.apps.geo.fixtures import AreaFactory, AreaTypeFactory from hct_mis_api.apps.geo.models import Country -from hct_mis_api.apps.household.fixtures import BankAccountInfoFactory, create_household +from hct_mis_api.apps.household.fixtures import ( + BankAccountInfoFactory, + create_household, + HouseholdFactory, + IndividualFactory, +) from hct_mis_api.apps.household.models import ( IDENTIFICATION_TYPE_NATIONAL_PASSPORT, IDENTIFICATION_TYPE_OTHER, @@ -30,6 +35,7 @@ def setUpTestData(cls) -> None: super().setUpTestData() create_afghanistan() cls.business_area = BusinessArea.objects.get(slug="afghanistan") + cls.program = ProgramFactory(business_area=cls.business_area) area_type_level_1 = AreaTypeFactory( name="State1", @@ -114,6 +120,12 @@ def test_remove_household(self) -> None: household2.delete(soft=False) self.assertIsNone(Household.all_objects.filter(unicef_id="HH-9191").first()) + def test_unique_unicef_id_per_program_constraint(self) -> None: + HouseholdFactory(unicef_id="HH-123", program=self.program) + HouseholdFactory(unicef_id="HH-000", program=self.program) + with self.assertRaises(IntegrityError): + HouseholdFactory(unicef_id="HH-123", program=self.program) + class TestDocument(TestCase): @classmethod @@ -503,8 +515,8 @@ class TestIndividualModel(TestCase): @classmethod def setUpTestData(cls) -> None: super().setUpTestData() - create_afghanistan() - ProgramFactory() + business_area = create_afghanistan() + cls.program = ProgramFactory(business_area=business_area) def test_bank_name(self) -> None: individual = create_household({"size": 1})[1][0] @@ -525,3 +537,9 @@ def test_bank_branch_name(self) -> None: individual = create_household({"size": 1})[1][0] bank_account_info = BankAccountInfoFactory(individual=individual) self.assertEqual(individual.bank_branch_name, bank_account_info.bank_branch_name) + + def test_unique_unicef_id_per_program_constraint(self) -> None: + IndividualFactory(unicef_id="IND-123", program=self.program) + IndividualFactory(unicef_id="IND-000", program=self.program) + with self.assertRaises(IntegrityError): + IndividualFactory(unicef_id="IND-123", program=self.program) From 455ec74c4f64fdd117d4f0c8a8ae75517766d0f7 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 19 Dec 2024 12:08:27 +0100 Subject: [PATCH 09/15] fix test --- tests/unit/apps/household/test_models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/apps/household/test_models.py b/tests/unit/apps/household/test_models.py index b8b81a3073..59821c5a60 100644 --- a/tests/unit/apps/household/test_models.py +++ b/tests/unit/apps/household/test_models.py @@ -442,6 +442,9 @@ def test_create_representations_duplicated_documents_with_different_numbers_and_ Document.objects.bulk_create(documents_to_create) # make representation with different number + (individual_to_create, _, _, _) = copy_individual_fast( + self.individual, program_3 + ) (program_3_individual_representation,) = Individual.objects.bulk_create([individual_to_create]) Document.objects.create( document_number="456", From 47ba8ae9e9d010294b3ff0648b15034a9cb4e10b Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 19 Dec 2024 12:08:57 +0100 Subject: [PATCH 10/15] remove one time scripts which are not valid with new contraint --- ...lation_import_incorrect_hh_ind_relation.py | 21 ---- ..._production_duplicates_after_enrollment.py | 50 --------- ...lation_import_incorrect_hh_ind_relation.py | 98 ----------------- ..._production_duplicates_after_enrollment.py | 103 ------------------ 4 files changed, 272 deletions(-) delete mode 100644 src/hct_mis_api/one_time_scripts/fix_program_population_import_incorrect_hh_ind_relation.py delete mode 100644 src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py delete mode 100644 tests/unit/one_time_scripts/test_fix_program_population_import_incorrect_hh_ind_relation.py delete mode 100644 tests/unit/one_time_scripts/test_remove_production_duplicates_after_enrollment.py diff --git a/src/hct_mis_api/one_time_scripts/fix_program_population_import_incorrect_hh_ind_relation.py b/src/hct_mis_api/one_time_scripts/fix_program_population_import_incorrect_hh_ind_relation.py deleted file mode 100644 index a265de4886..0000000000 --- a/src/hct_mis_api/one_time_scripts/fix_program_population_import_incorrect_hh_ind_relation.py +++ /dev/null @@ -1,21 +0,0 @@ -from django.db.models import F - -from hct_mis_api.apps.household.models import Household, Individual -from hct_mis_api.apps.registration_data.models import RegistrationDataImport - - -def fix_program_population_import_incorrect_hh_ind_relation() -> None: - individuals = Individual.all_objects.filter( - registration_data_import__data_source=RegistrationDataImport.PROGRAM_POPULATION, - ).exclude(household__registration_data_import=F("registration_data_import")) - - for individual in individuals: - household = Household.all_objects.filter( - registration_data_import=individual.registration_data_import, - copied_from_id=individual.household.copied_from_id, - copied_from__isnull=False, - program=individual.program, - ).first() - if household and household.unicef_id == individual.household.unicef_id: - individual.household = household - individual.save(update_fields=["household"]) diff --git a/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py b/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py deleted file mode 100644 index 7b4f5a4a5d..0000000000 --- a/src/hct_mis_api/one_time_scripts/remove_production_duplicates_after_enrollment.py +++ /dev/null @@ -1,50 +0,0 @@ -import logging - -from django.db.models import Count - -from hct_mis_api.apps.household.models import Household - -logger = logging.getLogger(__name__) - - -def remove_production_duplicates_after_enrollment() -> None: - # Exceptions from the further rules - for household in Household.objects.filter( - id__in=[ - "8b9bf768-4837-49aa-a598-5ad3c5822ca8", - "33a7bdf0-650d-49b4-b333-c49a7eb05356", - ] - ): - household.delete(soft=False) - - households_with_duplicates = ( - Household.objects.values("unicef_id", "program") - .annotate(household_count=Count("id")) - .filter(household_count__gt=1) - .order_by("copied_from__registration_data_import") - ) - logger.info(f"Found {households_with_duplicates.count()} households with duplicates") - - for i, entry in enumerate(households_with_duplicates, 1): - unicef_id = entry["unicef_id"] - program = entry["program"] - - households = Household.objects.filter(unicef_id=unicef_id, program=program).order_by("created_at") - - # Keep the first household and delete the duplicates - first = True - households_to_remove = [] - for household in households: - if first: - first = False - continue - if household.payment_set.exists(): - logger.info(f"Skipping {household.id} because it has payments") - continue - else: - households_to_remove.append(household) - for duplicate in households_to_remove: - duplicate.delete(soft=False) - - if i % 100 == 0: - logger.info(f"Processed {i}/{households_with_duplicates.count()} households") diff --git a/tests/unit/one_time_scripts/test_fix_program_population_import_incorrect_hh_ind_relation.py b/tests/unit/one_time_scripts/test_fix_program_population_import_incorrect_hh_ind_relation.py deleted file mode 100644 index dbce506524..0000000000 --- a/tests/unit/one_time_scripts/test_fix_program_population_import_incorrect_hh_ind_relation.py +++ /dev/null @@ -1,98 +0,0 @@ -from django.test import TestCase - -from hct_mis_api.apps.core.fixtures import create_afghanistan -from hct_mis_api.apps.household.fixtures import ( - HouseholdFactory, - create_household_and_individuals, -) -from hct_mis_api.apps.household.models import Household -from hct_mis_api.apps.program.fixtures import ProgramFactory -from hct_mis_api.apps.registration_data.fixtures import RegistrationDataImportFactory -from hct_mis_api.apps.registration_data.models import RegistrationDataImport -from hct_mis_api.one_time_scripts.fix_program_population_import_incorrect_hh_ind_relation import ( - fix_program_population_import_incorrect_hh_ind_relation, -) - - -class TestFixProgramPopulationIncorrectHhIndRelation(TestCase): - def test_fix_program_population_import_incorrect_hh_ind_relation(self) -> None: - business_area = create_afghanistan() - program = ProgramFactory(business_area=business_area) - copied_from_hh = HouseholdFactory(program=program) - program2 = ProgramFactory(business_area=business_area) - - rdi = RegistrationDataImportFactory(program=program2, data_source=RegistrationDataImport.PROGRAM_POPULATION) - rdi2 = RegistrationDataImportFactory(program=program2, data_source=RegistrationDataImport.PROGRAM_POPULATION) - rdi3 = RegistrationDataImportFactory(program=program2, data_source=RegistrationDataImport.PROGRAM_POPULATION) - household1, individuals1 = create_household_and_individuals( - { - "program": program2, - "unicef_id": "HH-01", - "rdi_merge_status": Household.PENDING, - "registration_data_import": rdi, - "copied_from": copied_from_hh, - }, - [ - {"rdi_merge_status": Household.PENDING}, - {"rdi_merge_status": Household.PENDING}, - ], - ) - - household2, individuals2 = create_household_and_individuals( - { - "program": program2, - "unicef_id": "HH-01", - "rdi_merge_status": Household.PENDING, - "registration_data_import": rdi2, - "copied_from": copied_from_hh, - }, - [{"rdi_merge_status": Household.PENDING}, {"rdi_merge_status": Household.PENDING}], - ) - ind2_1 = individuals2[0] - ind2_1.household = household1 - ind2_1.save() - - household3, individuals3 = create_household_and_individuals( - { - "program": program2, - "unicef_id": "HH-01", - "rdi_merge_status": Household.MERGED, - "registration_data_import": rdi3, - "copied_from": copied_from_hh, - }, - [{"rdi_merge_status": Household.MERGED}, {"rdi_merge_status": Household.MERGED}], - ) - ind3_1 = individuals3[0] - ind3_1.household = household1 - ind3_1.save() - ind3_2 = individuals3[1] - ind3_2.household = household2 - ind3_2.save() - - # check incorrect data - self.assertEqual(household1.individuals(manager="all_objects").count(), 4) - self.assertEqual(household2.individuals(manager="all_objects").count(), 2) - self.assertEqual(household3.individuals(manager="all_objects").count(), 0) - - self.assertNotEqual(ind2_1.household.registration_data_import, household2.registration_data_import) - self.assertNotEqual(ind3_1.household.registration_data_import, household3.registration_data_import) - self.assertNotEqual(ind3_2.household.registration_data_import, household3.registration_data_import) - - fix_program_population_import_incorrect_hh_ind_relation() - - ind2_1.refresh_from_db() - ind3_1.refresh_from_db() - ind3_2.refresh_from_db() - - self.assertEqual(household1.individuals(manager="all_objects").count(), 2) - - self.assertEqual(ind2_1.household, household2) - self.assertEqual(household2.individuals(manager="all_objects").count(), 2) - - self.assertEqual(ind3_1.household, household3) - self.assertEqual(ind3_2.household, household3) - self.assertEqual(household3.individuals(manager="all_objects").count(), 2) - - self.assertEqual(ind2_1.household.registration_data_import, household2.registration_data_import) - self.assertEqual(ind3_1.household.registration_data_import, household3.registration_data_import) - self.assertEqual(ind3_2.household.registration_data_import, household3.registration_data_import) diff --git a/tests/unit/one_time_scripts/test_remove_production_duplicates_after_enrollment.py b/tests/unit/one_time_scripts/test_remove_production_duplicates_after_enrollment.py deleted file mode 100644 index 0edcc9d0a0..0000000000 --- a/tests/unit/one_time_scripts/test_remove_production_duplicates_after_enrollment.py +++ /dev/null @@ -1,103 +0,0 @@ -from django.test import TestCase - -from hct_mis_api.apps.core.fixtures import create_afghanistan -from hct_mis_api.apps.household.fixtures import create_household_and_individuals -from hct_mis_api.apps.household.models import Household, Individual -from hct_mis_api.apps.payment.fixtures import PaymentFactory -from hct_mis_api.apps.program.fixtures import ProgramFactory -from hct_mis_api.one_time_scripts.remove_production_duplicates_after_enrollment import ( - remove_production_duplicates_after_enrollment, -) - - -class TestRemoveProductionDuplicatesAfterEnrollment(TestCase): - def test_remove_production_duplicates_after_enrollment(self) -> None: - business_area = create_afghanistan() - program = ProgramFactory(business_area=business_area) - program2 = ProgramFactory(business_area=business_area) - hh_unicef_id = "HH-20-0000.0001" - household_special_case1, individuals_special_case1 = create_household_and_individuals( - household_data={ - "id": "8b9bf768-4837-49aa-a598-5ad3c5822ca8", - "unicef_id": hh_unicef_id, - "business_area": program.business_area, - "program": program, - }, - individuals_data=[{}], - ) - household_special_case2, individuals_special_case2 = create_household_and_individuals( - household_data={ - "id": "33a7bdf0-650d-49b4-b333-c49a7eb05356", - "unicef_id": hh_unicef_id, - "business_area": program.business_area, - "program": program, - }, - individuals_data=[{}], - ) - household1, individuals1 = create_household_and_individuals( - household_data={ - "unicef_id": hh_unicef_id, - "business_area": program.business_area, - "program": program, - }, - individuals_data=[{}, {}], - ) - household2, individuals2 = create_household_and_individuals( - household_data={ - "unicef_id": hh_unicef_id, - "business_area": program.business_area, - "program": program, - }, - individuals_data=[{}, {}], - ) - household3, individuals3 = create_household_and_individuals( - household_data={ - "unicef_id": hh_unicef_id, - "business_area": program.business_area, - "program": program, - }, - individuals_data=[{}], - ) - PaymentFactory(household=household3) - - household4, individuals4 = create_household_and_individuals( - household_data={ - "unicef_id": hh_unicef_id, - "business_area": program.business_area, - "program": program, - }, - individuals_data=[{}], - ) - household_from_another_program, individuals_from_another_program = create_household_and_individuals( - household_data={ - "unicef_id": hh_unicef_id, - "business_area": program2.business_area, - "program": program2, - }, - individuals_data=[{}], - ) - - remove_production_duplicates_after_enrollment() - - self.assertIsNotNone(Household.all_objects.filter(id=household1.id).first()) - self.assertIsNotNone(Individual.all_objects.filter(id=individuals1[0].id).first()) - self.assertIsNotNone(Individual.all_objects.filter(id=individuals1[1].id).first()) - - self.assertIsNone(Household.all_objects.filter(id=household2.id).first()) - self.assertIsNone(Individual.all_objects.filter(id=individuals2[0].id).first()) - self.assertIsNone(Individual.all_objects.filter(id=individuals2[1].id).first()) - - self.assertIsNotNone(Individual.all_objects.filter(id=individuals3[0].id).first()) - self.assertIsNotNone(Household.all_objects.filter(id=household3.id).first()) - - self.assertIsNone(Household.all_objects.filter(id=household4.id).first()) - self.assertIsNone(Individual.all_objects.filter(id=individuals4[0].id).first()) - - self.assertIsNotNone(Household.all_objects.filter(id=household_from_another_program.id).first()) - self.assertIsNotNone(Individual.all_objects.filter(id=individuals_from_another_program[0].id).first()) - - self.assertIsNone(Household.all_objects.filter(id=household_special_case1.id).first()) - self.assertIsNone(Individual.all_objects.filter(id=individuals_special_case1[0].id).first()) - - self.assertIsNone(Household.all_objects.filter(id=household_special_case2.id).first()) - self.assertIsNone(Individual.all_objects.filter(id=individuals_special_case2[0].id).first()) From eba2c5e039254544c87ede468ed1c682a7d62f29 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 19 Dec 2024 12:09:55 +0100 Subject: [PATCH 11/15] fixed format --- tests/unit/apps/household/test_models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/apps/household/test_models.py b/tests/unit/apps/household/test_models.py index 59821c5a60..485a4fe534 100644 --- a/tests/unit/apps/household/test_models.py +++ b/tests/unit/apps/household/test_models.py @@ -9,9 +9,9 @@ from hct_mis_api.apps.geo.models import Country from hct_mis_api.apps.household.fixtures import ( BankAccountInfoFactory, - create_household, HouseholdFactory, IndividualFactory, + create_household, ) from hct_mis_api.apps.household.models import ( IDENTIFICATION_TYPE_NATIONAL_PASSPORT, @@ -442,9 +442,7 @@ def test_create_representations_duplicated_documents_with_different_numbers_and_ Document.objects.bulk_create(documents_to_create) # make representation with different number - (individual_to_create, _, _, _) = copy_individual_fast( - self.individual, program_3 - ) + (individual_to_create, _, _, _) = copy_individual_fast(self.individual, program_3) (program_3_individual_representation,) = Individual.objects.bulk_create([individual_to_create]) Document.objects.create( document_number="456", From 805e792569b1ed1340dce368bb3fae069f69cf75 Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 19 Dec 2024 12:35:46 +0100 Subject: [PATCH 12/15] change unique name --- src/hct_mis_api/apps/household/migrations/0005_migration.py | 6 +++--- src/hct_mis_api/apps/household/models.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/hct_mis_api/apps/household/migrations/0005_migration.py b/src/hct_mis_api/apps/household/migrations/0005_migration.py index ad31fcdc79..836f54f346 100644 --- a/src/hct_mis_api/apps/household/migrations/0005_migration.py +++ b/src/hct_mis_api/apps/household/migrations/0005_migration.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.25 on 2024-12-19 08:44 +# Generated by Django 3.2.25 on 2024-12-19 11:34 from django.db import migrations, models @@ -12,10 +12,10 @@ class Migration(migrations.Migration): operations = [ migrations.AddConstraint( model_name='household', - constraint=models.UniqueConstraint(condition=models.Q(('is_removed', False)), fields=('unicef_id', 'program'), name='unique_ind_unicef_id_in_program'), + constraint=models.UniqueConstraint(condition=models.Q(('is_removed', False)), fields=('unicef_id', 'program'), name='unique_hh_unicef_id_in_program'), ), migrations.AddConstraint( model_name='individual', - constraint=models.UniqueConstraint(condition=models.Q(('is_removed', False), ('duplicate', False)), fields=('unicef_id', 'program'), name='unique_hh_unicef_id_in_program'), + constraint=models.UniqueConstraint(condition=models.Q(('is_removed', False), ('duplicate', False)), fields=('unicef_id', 'program'), name='unique_ind_unicef_id_in_program'), ), ] diff --git a/src/hct_mis_api/apps/household/models.py b/src/hct_mis_api/apps/household/models.py index 39040bd2ed..0e0635e948 100644 --- a/src/hct_mis_api/apps/household/models.py +++ b/src/hct_mis_api/apps/household/models.py @@ -580,7 +580,7 @@ class Meta: UniqueConstraint( fields=["unicef_id", "program"], condition=Q(is_removed=False), - name="unique_ind_unicef_id_in_program", + name="unique_hh_unicef_id_in_program", ) ] @@ -1198,7 +1198,7 @@ class Meta: UniqueConstraint( fields=["unicef_id", "program"], condition=Q(is_removed=False) & Q(duplicate=False), - name="unique_hh_unicef_id_in_program", + name="unique_ind_unicef_id_in_program", ) ] From 807bfb01312f279d392ea2e45b22fc36ac53dec2 Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 19 Dec 2024 12:56:03 +0100 Subject: [PATCH 13/15] fixed tests --- .../test_registration_program_population_import_task.py | 2 +- .../snapshots/snap_test_copy_target_population_mutation.py | 2 +- .../apps/targeting/test_copy_target_population_mutation.py | 2 +- tests/unit/apps/targeting/test_targeting_validators.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unit/apps/registration_datahub/test_registration_program_population_import_task.py b/tests/unit/apps/registration_datahub/test_registration_program_population_import_task.py index 113474a75a..624520b69a 100644 --- a/tests/unit/apps/registration_datahub/test_registration_program_population_import_task.py +++ b/tests/unit/apps/registration_datahub/test_registration_program_population_import_task.py @@ -207,7 +207,7 @@ def test_registration_program_population_import_task(self) -> None: str(self.program_from.id), str(self.program_to.id), ) - self._imported_objects_count_after(2) + self._imported_objects_count_after(1) def test_registration_program_population_import_task_error(self) -> None: rdi_id = self.registration_data_import.id diff --git a/tests/unit/apps/targeting/snapshots/snap_test_copy_target_population_mutation.py b/tests/unit/apps/targeting/snapshots/snap_test_copy_target_population_mutation.py index dd1bcfdf15..364024e09f 100644 --- a/tests/unit/apps/targeting/snapshots/snap_test_copy_target_population_mutation.py +++ b/tests/unit/apps/targeting/snapshots/snap_test_copy_target_population_mutation.py @@ -151,7 +151,7 @@ 'status': 'OPEN', 'targetingCriteria': { 'householdIds': '', - 'individualIds': "['IND-1']", + 'individualIds': "['IND-12']", 'rules': [ ] }, diff --git a/tests/unit/apps/targeting/test_copy_target_population_mutation.py b/tests/unit/apps/targeting/test_copy_target_population_mutation.py index 21b2c081de..5ca5fcc072 100644 --- a/tests/unit/apps/targeting/test_copy_target_population_mutation.py +++ b/tests/unit/apps/targeting/test_copy_target_population_mutation.py @@ -86,7 +86,7 @@ def setUpTestData(cls) -> None: }, ) individual = individuals[0] - individual.unicef_id = "IND-1" + individual.unicef_id = "IND-12" individual.save() cls.household = household cls.update_partner_access_to_program(partner, cls.program) diff --git a/tests/unit/apps/targeting/test_targeting_validators.py b/tests/unit/apps/targeting/test_targeting_validators.py index 29d609085f..7dd0ba3e6d 100644 --- a/tests/unit/apps/targeting/test_targeting_validators.py +++ b/tests/unit/apps/targeting/test_targeting_validators.py @@ -46,17 +46,17 @@ def setUpTestData(cls) -> None: def test_TargetingCriteriaInputValidator(self) -> None: validator = TargetingCriteriaInputValidator - create_household({"unicef_id": "HH-1", "size": 1}, {"unicef_id": "IND-1"}) + create_household({"unicef_id": "HH-1", "size": 1}, {"unicef_id": "IND-12"}) self._update_program(self.program_standard) validator.validate( - {"rules": [{"Rule1": {"test": "123"}, "household_ids": "HH-1", "individual_ids": "IND-1"}]}, + {"rules": [{"Rule1": {"test": "123"}, "household_ids": "HH-1", "individual_ids": "IND-12"}]}, self.program_standard, ) with self.assertRaisesMessage(ValidationError, "Target criteria can only have individual ids"): self._update_program(self.program_standard_ind_only) validator.validate( - {"rules": [{"household_ids": "HH-1", "individual_ids": "IND-1"}]}, self.program_standard_ind_only + {"rules": [{"household_ids": "HH-1", "individual_ids": "IND-12"}]}, self.program_standard_ind_only ) with self.assertRaisesMessage(ValidationError, "There should be at least 1 rule in target criteria"): From 6221b0abcdf715169cf8ec5ea2584a74bf63e4cb Mon Sep 17 00:00:00 2001 From: Paulina Kujawa Date: Thu, 19 Dec 2024 13:06:45 +0100 Subject: [PATCH 14/15] fix more tests --- tests/unit/apps/registration_datahub/test_rdi_merge.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/apps/registration_datahub/test_rdi_merge.py b/tests/unit/apps/registration_datahub/test_rdi_merge.py index a58c99baed..739d295675 100644 --- a/tests/unit/apps/registration_datahub/test_rdi_merge.py +++ b/tests/unit/apps/registration_datahub/test_rdi_merge.py @@ -447,6 +447,7 @@ def test_merge_rdi_create_collections(self, household_representation_exists: boo if False, another household representation exists, but it does not have collection, if None, household representation does not exist in another program """ + program_2 = ProgramFactory(business_area=self.rdi.business_area) self.rdi.data_source = RegistrationDataImport.PROGRAM_POPULATION self.rdi.save() imported_household = HouseholdFactory( @@ -454,11 +455,13 @@ def test_merge_rdi_create_collections(self, household_representation_exists: boo registration_data_import=self.rdi, unicef_id="HH-9", rdi_merge_status=MergeStatusModel.PENDING, + program=self.rdi.program, ) self.set_imported_individuals(imported_household) individual_without_collection = IndividualFactory( unicef_id="IND-9", business_area=self.rdi.business_area, + program=program_2, household=None, ) individual_without_collection.individual_collection = None @@ -468,6 +471,7 @@ def test_merge_rdi_create_collections(self, household_representation_exists: boo IndividualFactory( unicef_id="IND-8", business_area=self.rdi.business_area, + program=program_2, individual_collection=individual_collection, household=None, ) @@ -477,6 +481,7 @@ def test_merge_rdi_create_collections(self, household_representation_exists: boo household = HouseholdFactory( head_of_household=individual_without_collection, business_area=self.rdi.business_area, + program=program_2, unicef_id="HH-9", ) household.household_collection = None From 3b5db8c9b20e920091efa2b18169275e909d35db Mon Sep 17 00:00:00 2001 From: Jan Romaniak Date: Thu, 19 Dec 2024 14:19:16 +0100 Subject: [PATCH 15/15] fix laast e2e tests --- tests/selenium/targeting/test_targeting.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/selenium/targeting/test_targeting.py b/tests/selenium/targeting/test_targeting.py index 3ecba9e3ea..8a699c13be 100644 --- a/tests/selenium/targeting/test_targeting.py +++ b/tests/selenium/targeting/test_targeting.py @@ -164,10 +164,13 @@ def create_flexible_attribute( return flexible_attribute -def create_custom_household(observed_disability: list[str], residence_status: str = HOST) -> Household: +def create_custom_household( + observed_disability: list[str], residence_status: str = HOST, unicef_id: str = "HH-00-0000.0442" +) -> Household: program = Program.objects.get(name="Test Programm") household, _ = create_household_and_individuals( household_data={ + "unicef_id": unicef_id, "rdi_merge_status": "MERGED", "business_area": program.business_area, "program": program, @@ -186,17 +189,17 @@ def create_custom_household(observed_disability: list[str], residence_status: st @pytest.fixture def household_with_disability() -> Household: - yield create_custom_household(observed_disability=[SEEING, HEARING]) + yield create_custom_household(observed_disability=[SEEING, HEARING], unicef_id="HH-00-0000.0443") @pytest.fixture def household_without_disabilities() -> Household: - yield create_custom_household(observed_disability=[]) + yield create_custom_household(observed_disability=[], unicef_id="HH-00-0000.0444") @pytest.fixture def household_refugee() -> Household: - yield create_custom_household(observed_disability=[], residence_status=REFUGEE) + yield create_custom_household(observed_disability=[], residence_status=REFUGEE, unicef_id="HH-00-0000.0445") def get_program_with_dct_type_and_name( @@ -234,6 +237,7 @@ def create_targeting(household_without_disabilities: Household) -> TargetPopulat target_population.save() household, _ = create_household( household_args={ + "unicef_id": "HH-00-0000.0440", "business_area": program.business_area, "program": program, "residence_status": HOST,