From 012715e175668d72e77fa39790b4c4cfdbceb096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pave=C5=82=20Ty=C5=9Blacki?= Date: Wed, 5 Jun 2024 23:27:01 +0100 Subject: [PATCH] fix idempotent mode tests --- CHANGES.md | 1 + ...2_relatedtesttable_relatedtesttable_idx.py | 4 +- .../idempotency_add_index_meta_app/models.py | 1 - ..._relatedtesttable_relatedtesttable_uniq.py | 1 - .../idempotency_add_unique_meta_app/models.py | 1 - tests/integration/__init__.py | 6 +- tests/integration/test_migrations.py | 83 ++++++++++++++----- 7 files changed, 67 insertions(+), 30 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 67225b7..5effcec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ - added `ZERO_DOWNTIME_MIGRATIONS_DROP_CONSTRAINS_BEFORE_CASCADE` settings and added dropping constraints and indexes before drop column or table - fixed sqlmigrate in idempotent mode - fixed creation unique constraint with include parameter + - fixed idempotent mode tests - updated unsafe migrations links to documentation - updated patched code to latest django version - updated test image to ubuntu 24.04 diff --git a/tests/apps/idempotency_add_index_meta_app/migrations/0002_relatedtesttable_relatedtesttable_idx.py b/tests/apps/idempotency_add_index_meta_app/migrations/0002_relatedtesttable_relatedtesttable_idx.py index 2145fbf..ecd38e4 100644 --- a/tests/apps/idempotency_add_index_meta_app/migrations/0002_relatedtesttable_relatedtesttable_idx.py +++ b/tests/apps/idempotency_add_index_meta_app/migrations/0002_relatedtesttable_relatedtesttable_idx.py @@ -13,9 +13,7 @@ class Migration(migrations.Migration): migrations.AddIndex( model_name="relatedtesttable", index=models.Index( - condition=models.Q(("test_field_int__gt", 0)), - fields=["test_field_int", "test_field_str"], - name="relatedtesttable_idx", + fields=["test_field_int", "test_field_str"], name="relatedtesttable_idx" ), ), ] diff --git a/tests/apps/idempotency_add_index_meta_app/models.py b/tests/apps/idempotency_add_index_meta_app/models.py index 4a8c834..d5348db 100644 --- a/tests/apps/idempotency_add_index_meta_app/models.py +++ b/tests/apps/idempotency_add_index_meta_app/models.py @@ -15,6 +15,5 @@ class Meta: models.Index( name="relatedtesttable_idx", fields=["test_field_int", "test_field_str"], - condition=models.Q(test_field_int__gt=0), ) ] diff --git a/tests/apps/idempotency_add_unique_meta_app/migrations/0002_relatedtesttable_relatedtesttable_uniq.py b/tests/apps/idempotency_add_unique_meta_app/migrations/0002_relatedtesttable_relatedtesttable_uniq.py index 7afee43..2936c34 100644 --- a/tests/apps/idempotency_add_unique_meta_app/migrations/0002_relatedtesttable_relatedtesttable_uniq.py +++ b/tests/apps/idempotency_add_unique_meta_app/migrations/0002_relatedtesttable_relatedtesttable_uniq.py @@ -13,7 +13,6 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="relatedtesttable", constraint=models.UniqueConstraint( - condition=models.Q(("test_field_int__gt", 0)), fields=("test_field_int", "test_field_str"), name="relatedtesttable_uniq", ), diff --git a/tests/apps/idempotency_add_unique_meta_app/models.py b/tests/apps/idempotency_add_unique_meta_app/models.py index 227bd58..0c96f9b 100644 --- a/tests/apps/idempotency_add_unique_meta_app/models.py +++ b/tests/apps/idempotency_add_unique_meta_app/models.py @@ -15,6 +15,5 @@ class Meta: models.UniqueConstraint( name="relatedtesttable_uniq", fields=["test_field_int", "test_field_str"], - condition=models.Q(test_field_int__gt=0), ) ] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index fc2be1e..e1f3af2 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -20,11 +20,11 @@ def pg_dump(table: str) -> str: host = settings.DATABASES["default"]["HOST"] port = settings.DATABASES["default"]["PORT"] name = settings.DATABASES["default"]["NAME"] - user = settings.DATABASES["default"]["USER"] - password = settings.DATABASES["default"]["PASSWORD"] + user = settings.DB_SUPER_USER + password = settings.DB_SUPER_PASSWORD env = os.environ.copy() | {"PGPASSWORD": password} cmd = f"pg_dump -h {host} -p {port} -U {user} -d {name} -s -t {table}" - popen = subprocess.run(cmd, env=env, text=True, shell=True, capture_output=True) + popen = subprocess.run(cmd, env=env, text=True, shell=True, capture_output=True, check=True) return popen.stdout diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 4aa69f8..ffe25fa 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -357,13 +357,17 @@ def test_idempotency_create_table(): ALTER TABLE "idempotency_create_table_app_relatedtesttable" DROP CONSTRAINT "idempotency_create_table_app_relatedtesttable_uniq"; """) + _drop_foreign_key_constraint_sql = one_line_sql(""" + SET CONSTRAINTS "idempotency_create_t_test_model_id_09b52f79_fk_idempoten" IMMEDIATE; + ALTER TABLE "idempotency_create_table_app_relatedtesttable" + DROP CONSTRAINT "idempotency_create_t_test_model_id_09b52f79_fk_idempoten"; + """) _drop_table_sql = one_line_sql(""" DROP TABLE "idempotency_create_table_app_relatedtesttable" CASCADE; """) # get target schema call_command("migrate", "idempotency_create_table_app", "0001") - old_schema = pg_dump("idempotency_create_table_app_relatedtesttable") call_command("migrate", "idempotency_create_table_app") new_schema = pg_dump("idempotency_create_table_app_relatedtesttable") @@ -467,6 +471,7 @@ def test_idempotency_create_table(): rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_create_table_app", "0002") assert split_sql_queries(rollback_sql) == [ _drop_unique_constraint_sql, + _drop_foreign_key_constraint_sql, _drop_table_sql, ] @@ -475,7 +480,6 @@ def test_idempotency_create_table(): with connection.cursor() as cursor: cursor.execute(_drop_unique_constraint_sql) call_command("migrate", "idempotency_create_table_app", "0001") - assert pg_dump("idempotency_create_table_app_relatedtesttable") == old_schema # rollback case 2 call_command("migrate", "idempotency_create_table_app") @@ -483,7 +487,6 @@ def test_idempotency_create_table(): cursor.execute(_drop_unique_constraint_sql) cursor.execute(_drop_table_sql) call_command("migrate", "idempotency_create_table_app", "0001") - assert pg_dump("idempotency_create_table_app_relatedtesttable") == old_schema @pytest.mark.django_db(transaction=True) @@ -571,6 +574,9 @@ def test_idempotency_add_column_foreign_key(): ON "idempotency_add_column_foreign_key_app_relatedtesttable" ("test_model_id"); """) + _drop_index_sql = one_line_sql(""" + DROP INDEX CONCURRENTLY IF EXISTS "idempotency_add_column_for_test_model_id_99eba75b"; + """) _drop_foreign_key_constraint_sql = one_line_sql(""" SET CONSTRAINTS "idempotency_add_colu_test_model_id_99eba75b_fk_idempoten" IMMEDIATE; ALTER TABLE "idempotency_add_column_foreign_key_app_relatedtesttable" @@ -664,6 +670,7 @@ def test_idempotency_add_column_foreign_key(): rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_add_column_foreign_key_app", "0002") assert split_sql_queries(rollback_sql) == [ _drop_foreign_key_constraint_sql, + _drop_index_sql, _drop_column_sql, ] @@ -717,6 +724,10 @@ def test_idempotency_add_column_one_to_one(): VALIDATE CONSTRAINT "idempotency_add_colu_test_model_id_3c5a49fe_fk_idempoten"; """) + _drop_unique_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_column_one_to_one_app_relatedtesttable" + DROP CONSTRAINT "idempotency_add_column_o_test_model_id_3c5a49fe_uniq"; + """) _drop_foreign_key_constraint_sql = one_line_sql(""" SET CONSTRAINTS "idempotency_add_colu_test_model_id_3c5a49fe_fk_idempoten" IMMEDIATE; ALTER TABLE "idempotency_add_column_one_to_one_app_relatedtesttable" @@ -820,6 +831,7 @@ def test_idempotency_add_column_one_to_one(): rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_add_column_one_to_one_app", "0002") assert split_sql_queries(rollback_sql) == [ _drop_foreign_key_constraint_sql, + _drop_unique_constraint_sql, _drop_column_sql, ] @@ -967,7 +979,7 @@ def test_idempotency_add_check(): call_command("migrate", "idempotency_add_check_app", "0001") old_schema = pg_dump("idempotency_add_check_app_relatedtesttable") call_command("migrate", "idempotency_add_check_app") - new_schema = pg_dump("idempotency_set_not_null_app_relatedtesttable") + new_schema = pg_dump("idempotency_add_check_app_relatedtesttable") # migrate call_command("migrate", "idempotency_add_check_app", "0001") @@ -1170,7 +1182,7 @@ def test_idempotency_add_one_to_one(): ALTER TABLE "idempotency_add_one_to_one_app_relatedtesttable" DROP CONSTRAINT "idempotency_add_one_to_o_test_field_int_8ebac681_uniq"; """) - _drop_index_sql = one_line_sql(""" + _drop_like_index_sql = one_line_sql(""" DROP INDEX CONCURRENTLY IF EXISTS "idempotency_add_one_to_o_test_field_int_8ebac681_like"; """) @@ -1255,7 +1267,7 @@ def test_idempotency_add_one_to_one(): assert split_sql_queries(rollback_sql) == [ _drop_foreign_key_constraint_sql, _drop_unique_constraint_sql, - _drop_index_sql, + _drop_like_index_sql, ] # rollback case 1 @@ -1278,7 +1290,7 @@ def test_idempotency_add_one_to_one(): with connection.cursor() as cursor: cursor.execute(_drop_foreign_key_constraint_sql) cursor.execute(_drop_unique_constraint_sql) - cursor.execute(_drop_index_sql) + cursor.execute(_drop_like_index_sql) call_command("migrate", "idempotency_add_one_to_one_app", "0001") assert pg_dump("idempotency_add_one_to_one_app_relatedtesttable") == old_schema @@ -1324,7 +1336,7 @@ def test_idempotency_add_index(): "idempotency_add_index_app__test_field_int_ecc428b5", ) call_command("migrate", "idempotency_add_index_app") - assert pg_dump("idempotency_add_unique_app_relatedtesttable") == new_schema + assert pg_dump("idempotency_add_index_app_relatedtesttable") == new_schema assert is_valid_index( "idempotency_add_index_app_relatedtesttable", "idempotency_add_index_app__test_field_int_ecc428b5", @@ -1372,8 +1384,7 @@ def test_idempotency_add_index(): def test_idempotency_add_index_meta(): _create_index_sql = one_line_sql(""" CREATE INDEX CONCURRENTLY "relatedtesttable_idx" - ON "idempotency_add_index_meta_app_relatedtesttable" ("test_field_int", "test_field_str") - WHERE "test_field_int" > 0; + ON "idempotency_add_index_meta_app_relatedtesttable" ("test_field_int", "test_field_str"); """) _drop_index_sql = one_line_sql(""" @@ -1463,7 +1474,7 @@ def test_idempotency_add_unique(): ALTER TABLE "idempotency_add_unique_app_relatedtesttable" DROP CONSTRAINT "idempotency_add_unique_a_test_field_int_01c4f0c0_uniq"; """) - _drop_unique_index_sql = one_line_sql(""" + _drop_like_index_sql = one_line_sql(""" DROP INDEX CONCURRENTLY IF EXISTS "idempotency_add_unique_a_test_field_int_01c4f0c0_like"; """) @@ -1526,7 +1537,7 @@ def test_idempotency_add_unique(): rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_add_unique_app", "0002") assert split_sql_queries(rollback_sql) == [ _drop_unique_constraint_sql, - _drop_unique_index_sql, + _drop_like_index_sql, ] # rollback case 1 @@ -1565,10 +1576,18 @@ def test_idempotency_add_unique(): def test_idempotency_add_unique_meta(): _create_unique_index_sql = one_line_sql(""" CREATE UNIQUE INDEX CONCURRENTLY "relatedtesttable_uniq" - ON "idempotency_add_unique_meta_app_relatedtesttable" ("test_field_int", "test_field_str") - WHERE "test_field_int" > 0; + ON "idempotency_add_unique_meta_app_relatedtesttable" ("test_field_int", "test_field_str"); + """) + _create_unique_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_unique_meta_app_relatedtesttable" + ADD CONSTRAINT "relatedtesttable_uniq" + UNIQUE USING INDEX "relatedtesttable_uniq"; """) + _drop_unique_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_unique_meta_app_relatedtesttable" + DROP CONSTRAINT "relatedtesttable_uniq"; + """) _drop_unique_index_sql = one_line_sql(""" DROP INDEX CONCURRENTLY IF EXISTS "relatedtesttable_uniq"; """) @@ -1585,6 +1604,7 @@ def test_idempotency_add_unique_meta(): migration_sql = call_command("sqlmigrate", "idempotency_add_unique_meta_app", "0002") assert split_sql_queries(migration_sql) == [ _create_unique_index_sql, + _create_unique_constraint_sql, ] # migrate case 1.1 @@ -1617,18 +1637,26 @@ def test_idempotency_add_unique_meta(): "relatedtesttable_uniq", ) + # migrate case 2 + call_command("migrate", "idempotency_add_unique_meta_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + call_command("migrate", "idempotency_add_unique_meta_app") + assert pg_dump("idempotency_add_unique_meta_app_relatedtesttable") == new_schema + # rollback (covers drop unique case) call_command("migrate", "idempotency_add_unique_meta_app") with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False): rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_add_unique_meta_app", "0002") assert split_sql_queries(rollback_sql) == [ - _drop_unique_index_sql, + _drop_unique_constraint_sql, ] # rollback case 1 call_command("migrate", "idempotency_add_unique_meta_app") with connection.cursor() as cursor: - cursor.execute(_drop_unique_index_sql) + cursor.execute(_drop_unique_constraint_sql) call_command("migrate", "idempotency_add_unique_meta_app", "0001") assert pg_dump("idempotency_add_unique_meta_app_relatedtesttable") == old_schema @@ -1798,13 +1826,26 @@ def test_idempotency_add_primary_key(): _create_unique_constraint_for_rollback_sql, ] + def old_schema_compatible(dump: str) -> str: + """ + django creates different name for primary key constraint than postgres + rolling back drop index can be reason of columns order changes + """ + return dump.replace( + "idempotency_add_primary_key_app_relatedtesttable_id_d0e5667c_pk", + "idempotency_add_primary_key_app_relatedtesttable_pkey", + ).replace( + "test_field_int integer NOT NULL,\n id integer NOT NULL", + "id integer NOT NULL,\n test_field_int integer NOT NULL", + ) + # rollback case 1 call_command("migrate", "idempotency_add_primary_key_app") with connection.cursor() as cursor: cursor.execute(_drop_primary_key_sql) with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): call_command("migrate", "idempotency_add_primary_key_app", "0001") - assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema + assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema # rollback case 2 call_command("migrate", "idempotency_add_primary_key_app") @@ -1813,7 +1854,7 @@ def test_idempotency_add_primary_key(): cursor.execute(_drop_unique_index_sql) with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): call_command("migrate", "idempotency_add_primary_key_app", "0001") - assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema + assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema # rollback case 3 call_command("migrate", "idempotency_add_primary_key_app") @@ -1823,7 +1864,7 @@ def test_idempotency_add_primary_key(): cursor.execute(_add_column_for_rollback_sql) with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): call_command("migrate", "idempotency_add_primary_key_app", "0001") - assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema + assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema # rollback case 4 call_command("migrate", "idempotency_add_primary_key_app") @@ -1834,7 +1875,7 @@ def test_idempotency_add_primary_key(): cursor.execute(_create_unique_index_for_rollback_sql) with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): call_command("migrate", "idempotency_add_primary_key_app", "0001") - assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema + assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema # rollback case 5 call_command("migrate", "idempotency_add_primary_key_app") @@ -1846,7 +1887,7 @@ def test_idempotency_add_primary_key(): cursor.execute(_create_unique_constraint_for_rollback_sql) with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): call_command("migrate", "idempotency_add_primary_key_app", "0001") - assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema + assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema @pytest.mark.django_db(transaction=True)