fix idempotent mode tests

CHANGES.md

@@ -7,6 +7,7 @@
  - added `ZERO_DOWNTIME_MIGRATIONS_DROP_CONSTRAINS_BEFORE_CASCADE` settings and added dropping constraints and indexes before drop column or table
  - fixed sqlmigrate in idempotent mode
  - fixed creation unique constraint with include parameter
+ - fixed idempotent mode tests
  - updated unsafe migrations links to documentation
  - updated patched code to latest django version
  - updated test image to ubuntu 24.04

tests/apps/idempotency_add_index_meta_app/migrations/0002_relatedtesttable_relatedtesttable_idx.py

@@ -13,9 +13,7 @@ class Migration(migrations.Migration):
  migrations.AddIndex(
  model_name="relatedtesttable",
  index=models.Index(
- condition=models.Q(("test_field_int__gt", 0)),
- fields=["test_field_int", "test_field_str"],
- name="relatedtesttable_idx",
+ fields=["test_field_int", "test_field_str"], name="relatedtesttable_idx"
  ),
  ),
  ]

tests/apps/idempotency_add_index_meta_app/models.py

@@ -15,6 +15,5 @@ class Meta:
  models.Index(
  name="relatedtesttable_idx",
  fields=["test_field_int", "test_field_str"],
- condition=models.Q(test_field_int__gt=0),
  )
  ]

tests/apps/idempotency_add_unique_meta_app/migrations/0002_relatedtesttable_relatedtesttable_uniq.py

@@ -13,7 +13,6 @@ class Migration(migrations.Migration):
  migrations.AddConstraint(
  model_name="relatedtesttable",
  constraint=models.UniqueConstraint(
- condition=models.Q(("test_field_int__gt", 0)),
  fields=("test_field_int", "test_field_str"),
  name="relatedtesttable_uniq",
  ),

tests/apps/idempotency_add_unique_meta_app/models.py

@@ -15,6 +15,5 @@ class Meta:
  models.UniqueConstraint(
  name="relatedtesttable_uniq",
  fields=["test_field_int", "test_field_str"],
- condition=models.Q(test_field_int__gt=0),
  )
  ]

tests/integration/__init__.py

@@ -20,11 +20,11 @@ def pg_dump(table: str) -> str:
  host = settings.DATABASES["default"]["HOST"]
  port = settings.DATABASES["default"]["PORT"]
  name = settings.DATABASES["default"]["NAME"]
- user = settings.DATABASES["default"]["USER"]
- password = settings.DATABASES["default"]["PASSWORD"]
+ user = settings.DB_SUPER_USER
+ password = settings.DB_SUPER_PASSWORD
  env = os.environ.copy() | {"PGPASSWORD": password}
  cmd = f"pg_dump -h {host} -p {port} -U {user} -d {name} -s -t {table}"
- popen = subprocess.run(cmd, env=env, text=True, shell=True, capture_output=True)
+ popen = subprocess.run(cmd, env=env, text=True, shell=True, capture_output=True, check=True)
  return popen.stdout
 
 

tests/integration/test_migrations.py

@@ -357,13 +357,17 @@ def test_idempotency_create_table():
  ALTER TABLE "idempotency_create_table_app_relatedtesttable"
  DROP CONSTRAINT "idempotency_create_table_app_relatedtesttable_uniq";
  """)
+ _drop_foreign_key_constraint_sql = one_line_sql("""
+ SET CONSTRAINTS "idempotency_create_t_test_model_id_09b52f79_fk_idempoten" IMMEDIATE;
+ ALTER TABLE "idempotency_create_table_app_relatedtesttable"
+ DROP CONSTRAINT "idempotency_create_t_test_model_id_09b52f79_fk_idempoten";
+ """)
  _drop_table_sql = one_line_sql("""
  DROP TABLE "idempotency_create_table_app_relatedtesttable" CASCADE;
  """)
 
  # get target schema
  call_command("migrate", "idempotency_create_table_app", "0001")
- old_schema = pg_dump("idempotency_create_table_app_relatedtesttable")
  call_command("migrate", "idempotency_create_table_app")
  new_schema = pg_dump("idempotency_create_table_app_relatedtesttable")
 
@@ -467,6 +471,7 @@ def test_idempotency_create_table():
  rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_create_table_app", "0002")
  assert split_sql_queries(rollback_sql) == [
  _drop_unique_constraint_sql,
+ _drop_foreign_key_constraint_sql,
  _drop_table_sql,
  ]
 
@@ -475,15 +480,13 @@ def test_idempotency_create_table():
  with connection.cursor() as cursor:
  cursor.execute(_drop_unique_constraint_sql)
  call_command("migrate", "idempotency_create_table_app", "0001")
- assert pg_dump("idempotency_create_table_app_relatedtesttable") == old_schema
 
  # rollback case 2
  call_command("migrate", "idempotency_create_table_app")
  with connection.cursor() as cursor:
  cursor.execute(_drop_unique_constraint_sql)
  cursor.execute(_drop_table_sql)
  call_command("migrate", "idempotency_create_table_app", "0001")
- assert pg_dump("idempotency_create_table_app_relatedtesttable") == old_schema
 
 
 @pytest.mark.django_db(transaction=True)
@@ -571,6 +574,9 @@ def test_idempotency_add_column_foreign_key():
  ON "idempotency_add_column_foreign_key_app_relatedtesttable" ("test_model_id");
  """)
 
+ _drop_index_sql = one_line_sql("""
+ DROP INDEX CONCURRENTLY IF EXISTS "idempotency_add_column_for_test_model_id_99eba75b";
+ """)
  _drop_foreign_key_constraint_sql = one_line_sql("""
  SET CONSTRAINTS "idempotency_add_colu_test_model_id_99eba75b_fk_idempoten" IMMEDIATE;
  ALTER TABLE "idempotency_add_column_foreign_key_app_relatedtesttable"
@@ -664,6 +670,7 @@ def test_idempotency_add_column_foreign_key():
  rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_add_column_foreign_key_app", "0002")
  assert split_sql_queries(rollback_sql) == [
  _drop_foreign_key_constraint_sql,
+ _drop_index_sql,
  _drop_column_sql,
  ]
 
@@ -717,6 +724,10 @@ def test_idempotency_add_column_one_to_one():
  VALIDATE CONSTRAINT "idempotency_add_colu_test_model_id_3c5a49fe_fk_idempoten";
  """)
 
+ _drop_unique_constraint_sql = one_line_sql("""
+ ALTER TABLE "idempotency_add_column_one_to_one_app_relatedtesttable"
+ DROP CONSTRAINT "idempotency_add_column_o_test_model_id_3c5a49fe_uniq";
+ """)
  _drop_foreign_key_constraint_sql = one_line_sql("""
  SET CONSTRAINTS "idempotency_add_colu_test_model_id_3c5a49fe_fk_idempoten" IMMEDIATE;
  ALTER TABLE "idempotency_add_column_one_to_one_app_relatedtesttable"
@@ -820,6 +831,7 @@ def test_idempotency_add_column_one_to_one():
  rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_add_column_one_to_one_app", "0002")
  assert split_sql_queries(rollback_sql) == [
  _drop_foreign_key_constraint_sql,
+ _drop_unique_constraint_sql,
  _drop_column_sql,
  ]
 
@@ -967,7 +979,7 @@ def test_idempotency_add_check():
  call_command("migrate", "idempotency_add_check_app", "0001")
  old_schema = pg_dump("idempotency_add_check_app_relatedtesttable")
  call_command("migrate", "idempotency_add_check_app")
- new_schema = pg_dump("idempotency_set_not_null_app_relatedtesttable")
+ new_schema = pg_dump("idempotency_add_check_app_relatedtesttable")
 
  # migrate
  call_command("migrate", "idempotency_add_check_app", "0001")
@@ -1170,7 +1182,7 @@ def test_idempotency_add_one_to_one():
  ALTER TABLE "idempotency_add_one_to_one_app_relatedtesttable"
  DROP CONSTRAINT "idempotency_add_one_to_o_test_field_int_8ebac681_uniq";
  """)
- _drop_index_sql = one_line_sql("""
+ _drop_like_index_sql = one_line_sql("""
  DROP INDEX CONCURRENTLY IF EXISTS "idempotency_add_one_to_o_test_field_int_8ebac681_like";
  """)
 
@@ -1255,7 +1267,7 @@ def test_idempotency_add_one_to_one():
  assert split_sql_queries(rollback_sql) == [
  _drop_foreign_key_constraint_sql,
  _drop_unique_constraint_sql,
- _drop_index_sql,
+ _drop_like_index_sql,
  ]
 
  # rollback case 1
@@ -1278,7 +1290,7 @@ def test_idempotency_add_one_to_one():
  with connection.cursor() as cursor:
  cursor.execute(_drop_foreign_key_constraint_sql)
  cursor.execute(_drop_unique_constraint_sql)
- cursor.execute(_drop_index_sql)
+ cursor.execute(_drop_like_index_sql)
  call_command("migrate", "idempotency_add_one_to_one_app", "0001")
  assert pg_dump("idempotency_add_one_to_one_app_relatedtesttable") == old_schema
 
@@ -1324,7 +1336,7 @@ def test_idempotency_add_index():
  "idempotency_add_index_app__test_field_int_ecc428b5",
  )
  call_command("migrate", "idempotency_add_index_app")
- assert pg_dump("idempotency_add_unique_app_relatedtesttable") == new_schema
+ assert pg_dump("idempotency_add_index_app_relatedtesttable") == new_schema
  assert is_valid_index(
  "idempotency_add_index_app_relatedtesttable",
  "idempotency_add_index_app__test_field_int_ecc428b5",
@@ -1372,8 +1384,7 @@ def test_idempotency_add_index():
 def test_idempotency_add_index_meta():
  _create_index_sql = one_line_sql("""
  CREATE INDEX CONCURRENTLY "relatedtesttable_idx"
- ON "idempotency_add_index_meta_app_relatedtesttable" ("test_field_int", "test_field_str")
- WHERE "test_field_int" > 0;
+ ON "idempotency_add_index_meta_app_relatedtesttable" ("test_field_int", "test_field_str");
  """)
 
  _drop_index_sql = one_line_sql("""
@@ -1463,7 +1474,7 @@ def test_idempotency_add_unique():
  ALTER TABLE "idempotency_add_unique_app_relatedtesttable"
  DROP CONSTRAINT "idempotency_add_unique_a_test_field_int_01c4f0c0_uniq";
  """)
- _drop_unique_index_sql = one_line_sql("""
+ _drop_like_index_sql = one_line_sql("""
  DROP INDEX CONCURRENTLY IF EXISTS "idempotency_add_unique_a_test_field_int_01c4f0c0_like";
  """)
 
@@ -1526,7 +1537,7 @@ def test_idempotency_add_unique():
  rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_add_unique_app", "0002")
  assert split_sql_queries(rollback_sql) == [
  _drop_unique_constraint_sql,
- _drop_unique_index_sql,
+ _drop_like_index_sql,
  ]
 
  # rollback case 1
@@ -1565,10 +1576,18 @@ def test_idempotency_add_unique():
 def test_idempotency_add_unique_meta():
  _create_unique_index_sql = one_line_sql("""
  CREATE UNIQUE INDEX CONCURRENTLY "relatedtesttable_uniq"
- ON "idempotency_add_unique_meta_app_relatedtesttable" ("test_field_int", "test_field_str")
- WHERE "test_field_int" > 0;
+ ON "idempotency_add_unique_meta_app_relatedtesttable" ("test_field_int", "test_field_str");
+ """)
+ _create_unique_constraint_sql = one_line_sql("""
+ ALTER TABLE "idempotency_add_unique_meta_app_relatedtesttable"
+ ADD CONSTRAINT "relatedtesttable_uniq"
+ UNIQUE USING INDEX "relatedtesttable_uniq";
  """)
 
+ _drop_unique_constraint_sql = one_line_sql("""
+ ALTER TABLE "idempotency_add_unique_meta_app_relatedtesttable"
+ DROP CONSTRAINT "relatedtesttable_uniq";
+ """)
  _drop_unique_index_sql = one_line_sql("""
  DROP INDEX CONCURRENTLY IF EXISTS "relatedtesttable_uniq";
  """)
@@ -1585,6 +1604,7 @@ def test_idempotency_add_unique_meta():
  migration_sql = call_command("sqlmigrate", "idempotency_add_unique_meta_app", "0002")
  assert split_sql_queries(migration_sql) == [
  _create_unique_index_sql,
+ _create_unique_constraint_sql,
  ]
 
  # migrate case 1.1
@@ -1617,18 +1637,26 @@ def test_idempotency_add_unique_meta():
  "relatedtesttable_uniq",
  )
 
+ # migrate case 2
+ call_command("migrate", "idempotency_add_unique_meta_app", "0001")
+ with connection.cursor() as cursor:
+ cursor.execute(_create_unique_index_sql)
+ cursor.execute(_create_unique_constraint_sql)
+ call_command("migrate", "idempotency_add_unique_meta_app")
+ assert pg_dump("idempotency_add_unique_meta_app_relatedtesttable") == new_schema
+
  # rollback (covers drop unique case)
  call_command("migrate", "idempotency_add_unique_meta_app")
  with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False):
  rollback_sql = call_command("sqlmigrate", "--backwards", "idempotency_add_unique_meta_app", "0002")
  assert split_sql_queries(rollback_sql) == [
- _drop_unique_index_sql,
+ _drop_unique_constraint_sql,
  ]
 
  # rollback case 1
  call_command("migrate", "idempotency_add_unique_meta_app")
  with connection.cursor() as cursor:
- cursor.execute(_drop_unique_index_sql)
+ cursor.execute(_drop_unique_constraint_sql)
  call_command("migrate", "idempotency_add_unique_meta_app", "0001")
  assert pg_dump("idempotency_add_unique_meta_app_relatedtesttable") == old_schema
 
@@ -1798,13 +1826,26 @@ def test_idempotency_add_primary_key():
  _create_unique_constraint_for_rollback_sql,
  ]
 
+ def old_schema_compatible(dump: str) -> str:
+ """
+ django creates different name for primary key constraint than postgres
+ rolling back drop index can be reason of columns order changes
+ """
+ return dump.replace(
+ "idempotency_add_primary_key_app_relatedtesttable_id_d0e5667c_pk",
+ "idempotency_add_primary_key_app_relatedtesttable_pkey",
+ ).replace(
+ "test_field_int integer NOT NULL,\n id integer NOT NULL",
+ "id integer NOT NULL,\n test_field_int integer NOT NULL",
+ )
+
  # rollback case 1
  call_command("migrate", "idempotency_add_primary_key_app")
  with connection.cursor() as cursor:
  cursor.execute(_drop_primary_key_sql)
  with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False):
  call_command("migrate", "idempotency_add_primary_key_app", "0001")
- assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema
+ assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema
 
  # rollback case 2
  call_command("migrate", "idempotency_add_primary_key_app")
@@ -1813,7 +1854,7 @@ def test_idempotency_add_primary_key():
  cursor.execute(_drop_unique_index_sql)
  with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False):
  call_command("migrate", "idempotency_add_primary_key_app", "0001")
- assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema
+ assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema
 
  # rollback case 3
  call_command("migrate", "idempotency_add_primary_key_app")
@@ -1823,7 +1864,7 @@ def test_idempotency_add_primary_key():
  cursor.execute(_add_column_for_rollback_sql)
  with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False):
  call_command("migrate", "idempotency_add_primary_key_app", "0001")
- assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema
+ assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema
 
  # rollback case 4
  call_command("migrate", "idempotency_add_primary_key_app")
@@ -1834,7 +1875,7 @@ def test_idempotency_add_primary_key():
  cursor.execute(_create_unique_index_for_rollback_sql)
  with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False):
  call_command("migrate", "idempotency_add_primary_key_app", "0001")
- assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema
+ assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema
 
  # rollback case 5
  call_command("migrate", "idempotency_add_primary_key_app")
@@ -1846,7 +1887,7 @@ def test_idempotency_add_primary_key():
  cursor.execute(_create_unique_constraint_for_rollback_sql)
  with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False):
  call_command("migrate", "idempotency_add_primary_key_app", "0001")
- assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == old_schema
+ assert old_schema_compatible(pg_dump("idempotency_add_primary_key_app_relatedtesttable")) == old_schema
 
 
 @pytest.mark.django_db(transaction=True)