From f41984bf3f3f970cca22a6b0166074cd09cbd65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pave=C5=82=20Ty=C5=9Blacki?= Date: Wed, 24 Apr 2024 17:21:11 +0100 Subject: [PATCH] add idempotent mode --- CHANGES.md | 3 + README.md | 19 +- .../backends/postgres/schema.py | 363 ++++++++-- .../idempotency_add_check_app/__init__.py | 0 .../migrations/0001_initial.py | 42 ++ ...cy_add_check_app_relatedtesttable_check.py | 20 + .../migrations/__init__.py | 0 .../apps/idempotency_add_check_app/models.py | 18 + .../idempotency_add_column_app/__init__.py | 0 .../migrations/0001_initial.py | 42 ++ .../0002_relatedtesttable_test_model.py | 25 + .../migrations/__init__.py | 0 .../apps/idempotency_add_column_app/models.py | 11 + .../__init__.py | 0 .../migrations/0001_initial.py | 42 ++ ...2_alter_relatedtesttable_test_field_int.py | 26 + .../migrations/__init__.py | 0 .../idempotency_add_foreign_key_app/models.py | 15 + .../__init__.py | 0 .../migrations/0001_initial.py | 42 ++ ...002_remove_relatedtesttable_id_and_more.py | 22 + .../migrations/__init__.py | 0 .../idempotency_add_primary_key_app/models.py | 10 + .../idempotency_add_unique_app/__init__.py | 0 .../migrations/0001_initial.py | 42 ++ ...2_alter_relatedtesttable_test_field_int.py | 18 + .../migrations/__init__.py | 0 .../apps/idempotency_add_unique_app/models.py | 10 + .../idempotency_create_table_app/__init__.py | 0 .../migrations/0001_initial.py | 26 + .../0002_relatedtesttable_and_more.py | 46 ++ .../migrations/__init__.py | 0 .../idempotency_create_table_app/models.py | 19 + .../idempotency_set_not_null_app/__init__.py | 0 .../migrations/0001_initial.py | 42 ++ ...2_alter_relatedtesttable_test_field_int.py | 18 + .../migrations/__init__.py | 0 .../idempotency_set_not_null_app/models.py | 10 + tests/integration/__init__.py | 49 ++ tests/integration/test_migrations.py | 675 +++++++++++++++++- tests/settings_make_migrations.py | 9 +- 41 files changed, 1604 insertions(+), 60 deletions(-) create mode 100644 tests/apps/idempotency_add_check_app/__init__.py create mode 100644 tests/apps/idempotency_add_check_app/migrations/0001_initial.py create mode 100644 tests/apps/idempotency_add_check_app/migrations/0002_relatedtesttable_idempotency_add_check_app_relatedtesttable_check.py create mode 100644 tests/apps/idempotency_add_check_app/migrations/__init__.py create mode 100644 tests/apps/idempotency_add_check_app/models.py create mode 100644 tests/apps/idempotency_add_column_app/__init__.py create mode 100644 tests/apps/idempotency_add_column_app/migrations/0001_initial.py create mode 100644 tests/apps/idempotency_add_column_app/migrations/0002_relatedtesttable_test_model.py create mode 100644 tests/apps/idempotency_add_column_app/migrations/__init__.py create mode 100644 tests/apps/idempotency_add_column_app/models.py create mode 100644 tests/apps/idempotency_add_foreign_key_app/__init__.py create mode 100644 tests/apps/idempotency_add_foreign_key_app/migrations/0001_initial.py create mode 100644 tests/apps/idempotency_add_foreign_key_app/migrations/0002_alter_relatedtesttable_test_field_int.py create mode 100644 tests/apps/idempotency_add_foreign_key_app/migrations/__init__.py create mode 100644 tests/apps/idempotency_add_foreign_key_app/models.py create mode 100644 tests/apps/idempotency_add_primary_key_app/__init__.py create mode 100644 tests/apps/idempotency_add_primary_key_app/migrations/0001_initial.py create mode 100644 tests/apps/idempotency_add_primary_key_app/migrations/0002_remove_relatedtesttable_id_and_more.py create mode 100644 tests/apps/idempotency_add_primary_key_app/migrations/__init__.py create mode 100644 tests/apps/idempotency_add_primary_key_app/models.py create mode 100644 tests/apps/idempotency_add_unique_app/__init__.py create mode 100644 tests/apps/idempotency_add_unique_app/migrations/0001_initial.py create mode 100644 tests/apps/idempotency_add_unique_app/migrations/0002_alter_relatedtesttable_test_field_int.py create mode 100644 tests/apps/idempotency_add_unique_app/migrations/__init__.py create mode 100644 tests/apps/idempotency_add_unique_app/models.py create mode 100644 tests/apps/idempotency_create_table_app/__init__.py create mode 100644 tests/apps/idempotency_create_table_app/migrations/0001_initial.py create mode 100644 tests/apps/idempotency_create_table_app/migrations/0002_relatedtesttable_and_more.py create mode 100644 tests/apps/idempotency_create_table_app/migrations/__init__.py create mode 100644 tests/apps/idempotency_create_table_app/models.py create mode 100644 tests/apps/idempotency_set_not_null_app/__init__.py create mode 100644 tests/apps/idempotency_set_not_null_app/migrations/0001_initial.py create mode 100644 tests/apps/idempotency_set_not_null_app/migrations/0002_alter_relatedtesttable_test_field_int.py create mode 100644 tests/apps/idempotency_set_not_null_app/migrations/__init__.py create mode 100644 tests/apps/idempotency_set_not_null_app/models.py diff --git a/CHANGES.md b/CHANGES.md index 82cba13..5e26217 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,8 @@ # django-pg-zero-downtime-migrations changelog +## 0.15 + - added idempotent mode and `ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL` setting + ## 0.14 - fixed deferred sql errors - added django 5.0 support diff --git a/README.md b/README.md index 65d3fcd..1868b91 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,23 @@ Define way to apply deferred sql, default `True`: ZERO_DOWNTIME_DEFERRED_SQL = True Allowed values: -- `True` - run deferred sql similar to default django way. -- `False` - run deferred sql as soon as possible. +- `True` - run deferred sql similar to default django way +- `False` - run deferred sql as soon as possible + +#### ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL + +Define idempotent mode, default `False`: + + ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL = False + +Allowed values: +- `True` - skip already applied sql migrations +- `False` - standard non atomic django behaviour + +As this backend doesn't use transactions for migrations any failed migration can be cause of stopped process in intermediate state. +To avoid manual schema manipulation idempotent mode allows to rerun failed migration after fixed issue (eg. data issue or long running CRUD queries). + +> _NOTE:_ idempotent mode checks rely only on name and index and constraint valid state, so it can ignore name collisions and recommended do not use it for CI checks. #### PgBouncer and timeouts diff --git a/django_zero_downtime_migrations/backends/postgres/schema.py b/django_zero_downtime_migrations/backends/postgres/schema.py index efce6ab..5b4cbce 100644 --- a/django_zero_downtime_migrations/backends/postgres/schema.py +++ b/django_zero_downtime_migrations/backends/postgres/schema.py @@ -69,6 +69,33 @@ def format(self, *args, **kwargs): DUMMY_SQL = DummySQL() +class Condition: + def __init__(self, sql, exists, idempotent_mode_only=False): + self.sql = sql + self.exists = exists + self.idempotent_mode_only = idempotent_mode_only + + def __str__(self): + return self.sql + + def __repr__(self): + return str(self) + + def __mod__(self, other): + return self.__class__( + sql=self.sql % other, + exists=self.exists, + idempotent_mode_only=self.idempotent_mode_only, + ) + + def format(self, *args, **kwargs): + return self.__class__( + sql=self.sql.format(*args, **kwargs), + exists=self.exists, + idempotent_mode_only=self.idempotent_mode_only, + ) + + class MultiStatementSQL(list): def __init__(self, obj, *args): @@ -99,12 +126,20 @@ def format(self, *args, **kwargs): class PGLock: - def __init__(self, sql, use_timeouts=False, disable_statement_timeout=False): + def __init__( + self, + sql, + *, + use_timeouts=False, + disable_statement_timeout=False, + idempotent_condition=None, + ): self.sql = sql if use_timeouts and disable_statement_timeout: raise ValueError("Can't apply use_timeouts and disable_statement_timeout simultaneously.") self.use_timeouts = use_timeouts self.disable_statement_timeout = disable_statement_timeout + self.idempotent_condition = idempotent_condition def __str__(self): return self.sql @@ -119,22 +154,60 @@ def __mod__(self, other): return DUMMY_SQL if isinstance(other, dict) and any(val is DUMMY_SQL for val in other.values()): return DUMMY_SQL - return self.__class__(self.sql % other, self.use_timeouts, self.disable_statement_timeout) + return self.__class__( + self.sql % other, + use_timeouts=self.use_timeouts, + disable_statement_timeout=self.disable_statement_timeout, + idempotent_condition=self.idempotent_condition % other + if self.idempotent_condition is not None else None, + ) def format(self, *args, **kwargs): if any(arg is DUMMY_SQL for arg in args) or any(val is DUMMY_SQL for val in kwargs.values()): return DUMMY_SQL - return self.__class__(self.sql.format(*args, **kwargs), self.use_timeouts, self.disable_statement_timeout) + return self.__class__( + self.sql.format(*args, **kwargs), + use_timeouts=self.use_timeouts, + disable_statement_timeout=self.disable_statement_timeout, + idempotent_condition=self.idempotent_condition.format(*args, **kwargs) + if self.idempotent_condition is not None else None, + ) class PGAccessExclusive(PGLock): - def __init__(self, sql, use_timeouts=True, disable_statement_timeout=False): - super().__init__(sql, use_timeouts, disable_statement_timeout) + def __init__( + self, + sql, + *, + use_timeouts=True, + disable_statement_timeout=False, + idempotent_condition=None, + ): + super().__init__( + sql, + use_timeouts=use_timeouts, + disable_statement_timeout=disable_statement_timeout, + idempotent_condition=idempotent_condition, + ) class PGShareUpdateExclusive(PGLock): - pass + + def __init__( + self, + sql, + *, + use_timeouts=False, + disable_statement_timeout=True, + idempotent_condition=None, + ): + super().__init__( + sql, + use_timeouts=use_timeouts, + disable_statement_timeout=disable_statement_timeout, + idempotent_condition=idempotent_condition, + ) class DatabaseSchemaEditorMixin: @@ -145,6 +218,40 @@ class DatabaseSchemaEditorMixin: sql_set_lock_timeout = "SET lock_timeout TO '%(lock_timeout)s'" sql_set_statement_timeout = "SET statement_timeout TO '%(statement_timeout)s'" + # _sql_sequence_exists = "SELECT 1 FROM pg_class WHERE relname = TRIM('\"' FROM '%(table)s')" + _sql_index_exists = "SELECT 1 FROM pg_class WHERE relname = TRIM('\"' FROM '%(name)s')" + _sql_table_exists = "SELECT 1 FROM pg_class WHERE relname = TRIM('\"' FROM '%(table)s')" + _sql_new_table_exists = "SELECT 1 FROM pg_class WHERE relname = TRIM('\"' FROM '%(new_table)s')" + _sql_column_exists = ( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = TRIM('\"' FROM '%(table)s') " + "AND column_name = TRIM('\"' FROM '%(column)s')" + ) + _sql_new_column_exists = ( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = TRIM('\"' FROM '%(table)s') " + "AND column_name = TRIM('\"' FROM '%(new_column)s')" + ) + _sql_constraint_exists = ( + "SELECT 1 FROM information_schema.table_constraints " + "WHERE table_name = TRIM('\"' FROM '%(table)s') " + "AND constraint_name = TRIM('\"' FROM '%(name)s')" + ) + _sql_index_valid = ( + "SELECT 1 " + "FROM pg_index " + "WHERE indrelid = TRIM('\"' FROM '%(table)s')::regclass::oid " + "AND indexrelid = TRIM('\"' FROM '%(name)s')::regclass::oid " + "AND indisvalid" + ) + _sql_constraint_valid = ( + "SELECT 1 " + "FROM pg_constraint " + "WHERE conrelid = TRIM('\"' FROM '%(table)s')::regclass::oid " + "AND conname = TRIM('\"' FROM '%(name)s') " + "AND convalidated" + ) + if django.VERSION[:2] >= (4, 1): sql_alter_sequence_type = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_alter_sequence_type) sql_add_identity = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_add_identity) @@ -154,72 +261,185 @@ class DatabaseSchemaEditorMixin: sql_create_sequence = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_create_sequence) sql_set_sequence_owner = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_set_sequence_owner) sql_delete_sequence = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_delete_sequence) - sql_create_table = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_create_table, use_timeouts=False) - sql_delete_table = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_delete_table, use_timeouts=False) + sql_create_table = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_create_table, + idempotent_condition=Condition(_sql_table_exists, False), + use_timeouts=False, + ) + + sql_delete_table = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_delete_table, + idempotent_condition=Condition(_sql_table_exists, True), + use_timeouts=False, + ) - sql_rename_table = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_rename_table) + sql_rename_table = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_rename_table, + idempotent_condition=Condition(_sql_new_table_exists, False), + ) sql_retablespace_table = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_retablespace_table) sql_create_column_inline_fk = None - sql_create_column = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_create_column) + sql_create_column = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_create_column, + idempotent_condition=Condition(_sql_column_exists, False), + ) sql_alter_column = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_alter_column) - sql_delete_column = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_delete_column) - sql_rename_column = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_rename_column) + sql_delete_column = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_delete_column, + idempotent_condition=Condition(_sql_column_exists, True), + ) + sql_rename_column = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_rename_column, + idempotent_condition=Condition(_sql_new_column_exists, False), + ) sql_create_check = MultiStatementSQL( - PGAccessExclusive("ALTER TABLE %(table)s ADD CONSTRAINT %(name)s CHECK (%(check)s) NOT VALID"), - PGShareUpdateExclusive("ALTER TABLE %(table)s VALIDATE CONSTRAINT %(name)s", - disable_statement_timeout=True), + PGAccessExclusive( + "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s CHECK (%(check)s) NOT VALID", + idempotent_condition=Condition(_sql_constraint_exists, False), + ), + PGShareUpdateExclusive( + "ALTER TABLE %(table)s VALIDATE CONSTRAINT %(name)s", + disable_statement_timeout=True, + ), ) - sql_delete_check = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_delete_check) + sql_delete_check = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_delete_check, + idempotent_condition=Condition(_sql_constraint_exists, True), + ) + + _sql_reindex_for_idempotent_mode = PGShareUpdateExclusive( + "REINDEX INDEX CONCURRENTLY %(name)s", + idempotent_condition=Condition(_sql_index_valid, False, idempotent_mode_only=True), + disable_statement_timeout=True, + ), if django.VERSION[:2] >= (5, 0): sql_create_unique = MultiStatementSQL( - PGShareUpdateExclusive("CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s " - "(%(columns)s)%(nulls_distinct)s", - disable_statement_timeout=True), - PGAccessExclusive("ALTER TABLE %(table)s ADD CONSTRAINT %(name)s UNIQUE USING INDEX %(name)s"), + PGShareUpdateExclusive( + "CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)%(nulls_distinct)s", + idempotent_condition=Condition(_sql_index_exists, False), + disable_statement_timeout=True, + ), + PGShareUpdateExclusive( + "REINDEX INDEX CONCURRENTLY %(name)s", + idempotent_condition=Condition(_sql_index_valid, False, idempotent_mode_only=True), + disable_statement_timeout=True, + ), + PGAccessExclusive( + "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s UNIQUE USING INDEX %(name)s", + idempotent_condition=Condition(_sql_constraint_exists, False), + ), ) else: sql_create_unique = MultiStatementSQL( - PGShareUpdateExclusive("CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)", - disable_statement_timeout=True), - PGAccessExclusive("ALTER TABLE %(table)s ADD CONSTRAINT %(name)s UNIQUE USING INDEX %(name)s"), + PGShareUpdateExclusive( + "CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)", + idempotent_condition=Condition(_sql_index_exists, False), + disable_statement_timeout=True, + ), + PGShareUpdateExclusive( + "REINDEX INDEX CONCURRENTLY %(name)s", + idempotent_condition=Condition(_sql_index_valid, False, idempotent_mode_only=True), + disable_statement_timeout=True, + ), + PGAccessExclusive( + "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s UNIQUE USING INDEX %(name)s", + idempotent_condition=Condition(_sql_constraint_exists, False), + ), ) - sql_delete_unique = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_delete_unique) + sql_delete_unique = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_delete_unique, + idempotent_condition=Condition(_sql_constraint_exists, True), + ) sql_create_fk = MultiStatementSQL( - PGAccessExclusive("ALTER TABLE %(table)s ADD CONSTRAINT %(name)s FOREIGN KEY (%(column)s) " - "REFERENCES %(to_table)s (%(to_column)s)%(deferrable)s NOT VALID"), - PGShareUpdateExclusive("ALTER TABLE %(table)s VALIDATE CONSTRAINT %(name)s", - disable_statement_timeout=True), + PGAccessExclusive( + "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s FOREIGN KEY (%(column)s) " + "REFERENCES %(to_table)s (%(to_column)s)%(deferrable)s NOT VALID", + idempotent_condition=Condition(_sql_constraint_exists, False), + ), + PGShareUpdateExclusive( + "ALTER TABLE %(table)s VALIDATE CONSTRAINT %(name)s", + disable_statement_timeout=True, + ), + ) + sql_delete_fk = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_delete_fk, + idempotent_condition=Condition(_sql_constraint_exists, True), ) - sql_delete_fk = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_delete_fk) sql_create_pk = MultiStatementSQL( - PGShareUpdateExclusive("CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)", - disable_statement_timeout=True), - PGAccessExclusive("ALTER TABLE %(table)s ADD CONSTRAINT %(name)s PRIMARY KEY USING INDEX %(name)s"), + PGShareUpdateExclusive( + "CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)", + idempotent_condition=Condition(_sql_index_exists, False), + disable_statement_timeout=True, + ), + PGShareUpdateExclusive( + "REINDEX INDEX CONCURRENTLY %(name)s", + idempotent_condition=Condition(_sql_index_valid, False, idempotent_mode_only=True), + disable_statement_timeout=True, + ), + PGAccessExclusive( + "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s PRIMARY KEY USING INDEX %(name)s", + idempotent_condition=Condition(_sql_constraint_exists, False), + ), + ) + sql_delete_pk = PGAccessExclusive( + PostgresDatabaseSchemaEditor.sql_delete_pk, + idempotent_condition=Condition(_sql_constraint_exists, True), ) - sql_delete_pk = PGAccessExclusive(PostgresDatabaseSchemaEditor.sql_delete_pk) - sql_create_index = PGShareUpdateExclusive( - PostgresDatabaseSchemaEditor.sql_create_index_concurrently, - disable_statement_timeout=True + sql_create_index = MultiStatementSQL( + PGShareUpdateExclusive( + PostgresDatabaseSchemaEditor.sql_create_index_concurrently, + idempotent_condition=Condition(_sql_index_exists, False), + disable_statement_timeout=True, + ), + PGShareUpdateExclusive( + "REINDEX INDEX CONCURRENTLY %(name)s", + idempotent_condition=Condition(_sql_index_valid, False, idempotent_mode_only=True), + disable_statement_timeout=True, + ), ) - sql_create_index_concurrently = PGShareUpdateExclusive( - PostgresDatabaseSchemaEditor.sql_create_index_concurrently, - disable_statement_timeout=True + sql_create_index_concurrently = MultiStatementSQL( + PGShareUpdateExclusive( + PostgresDatabaseSchemaEditor.sql_create_index_concurrently, + idempotent_condition=Condition(_sql_index_exists, False), + disable_statement_timeout=True, + ), + PGShareUpdateExclusive( + "REINDEX INDEX CONCURRENTLY %(name)s", + idempotent_condition=Condition(_sql_index_valid, False, idempotent_mode_only=True), + disable_statement_timeout=True, + ), ) if django.VERSION[:2] >= (5, 0): - sql_create_unique_index = PGShareUpdateExclusive( - "CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)%(condition)s%(nulls_distinct)s", - disable_statement_timeout=True + sql_create_unique_index = MultiStatementSQL( + PGShareUpdateExclusive( + "CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)%(condition)s%(nulls_distinct)s", + idempotent_condition=Condition(_sql_index_exists, False), + disable_statement_timeout=True, + ), + PGShareUpdateExclusive( + "REINDEX INDEX CONCURRENTLY %(name)s", + idempotent_condition=Condition(_sql_index_valid, False, idempotent_mode_only=True), + disable_statement_timeout=True, + ), ) else: - sql_create_unique_index = PGShareUpdateExclusive( - "CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)%(condition)s", - disable_statement_timeout=True + sql_create_unique_index = MultiStatementSQL( + PGShareUpdateExclusive( + "CREATE UNIQUE INDEX CONCURRENTLY %(name)s ON %(table)s (%(columns)s)%(condition)s", + idempotent_condition=Condition(_sql_index_exists, False), + disable_statement_timeout=True, + ), + PGShareUpdateExclusive( + "REINDEX INDEX CONCURRENTLY %(name)s", + idempotent_condition=Condition(_sql_index_valid, False, idempotent_mode_only=True), + disable_statement_timeout=True, + ), ) sql_delete_index = PGShareUpdateExclusive("DROP INDEX CONCURRENTLY IF EXISTS %(name)s") sql_delete_index_concurrently = PGShareUpdateExclusive( @@ -231,11 +451,19 @@ class DatabaseSchemaEditorMixin: sql_alter_column_comment = PGShareUpdateExclusive(PostgresDatabaseSchemaEditor.sql_alter_column_comment) _sql_column_not_null = MultiStatementSQL( - PGAccessExclusive("ALTER TABLE %(table)s ADD CONSTRAINT %(name)s CHECK (%(column)s IS NOT NULL) NOT VALID"), - PGShareUpdateExclusive("ALTER TABLE %(table)s VALIDATE CONSTRAINT %(name)s", - disable_statement_timeout=True), + PGAccessExclusive( + "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s CHECK (%(column)s IS NOT NULL) NOT VALID", + idempotent_condition=Condition(_sql_constraint_exists, False), + ), + PGShareUpdateExclusive( + "ALTER TABLE %(table)s VALIDATE CONSTRAINT %(name)s", + disable_statement_timeout=True, + ), PGAccessExclusive("ALTER TABLE %(table)s ALTER COLUMN %(column)s SET NOT NULL"), - PGAccessExclusive("ALTER TABLE %(table)s DROP CONSTRAINT %(name)s"), + PGAccessExclusive( + "ALTER TABLE %(table)s DROP CONSTRAINT %(name)s", + idempotent_condition=Condition(_sql_constraint_exists, True), + ), ) _varchar_type_regexp = re.compile(r'^varchar\((?P\d+)\)$') @@ -256,6 +484,7 @@ def __init__(self, connection, collect_sql=False, atomic=True): settings, "ZERO_DOWNTIME_MIGRATIONS_FLEXIBLE_STATEMENT_TIMEOUT", False) self.RAISE_FOR_UNSAFE = getattr(settings, "ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE", False) self.DEFERRED_SQL = getattr(settings, "ZERO_DOWNTIME_MIGRATIONS_DEFERRED_SQL", True) + self.IDEMPOTENT_SQL = getattr(settings, "ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL", False) def execute(self, sql, params=()): if sql is DUMMY_SQL: @@ -268,26 +497,48 @@ def execute(self, sql, params=()): else: statements.append(sql) for statement in statements: + idempotent_condition = None if isinstance(statement, PGLock): use_timeouts = statement.use_timeouts disable_statement_timeout = statement.disable_statement_timeout + idempotent_condition = statement.idempotent_condition statement = statement.sql elif isinstance(statement, Statement) and isinstance(statement.template, PGLock): use_timeouts = statement.template.use_timeouts disable_statement_timeout = statement.template.disable_statement_timeout + if statement.template.idempotent_condition is not None: + idempotent_condition = statement.template.idempotent_condition % statement.parts statement = Statement(statement.template.sql, **statement.parts) else: use_timeouts = False disable_statement_timeout = False - if use_timeouts: - with self._set_operation_timeout(self.STATEMENT_TIMEOUT, self.LOCK_TIMEOUT): - super().execute(statement, params) - elif disable_statement_timeout and self.FLEXIBLE_STATEMENT_TIMEOUT: - with self._set_operation_timeout(self.ZERO_TIMEOUT): + if not self._skip_applied(idempotent_condition): + if use_timeouts: + with self._set_operation_timeout(self.STATEMENT_TIMEOUT, self.LOCK_TIMEOUT): + super().execute(statement, params) + elif disable_statement_timeout and self.FLEXIBLE_STATEMENT_TIMEOUT: + with self._set_operation_timeout(self.ZERO_TIMEOUT): + super().execute(statement, params) + else: super().execute(statement, params) - else: - super().execute(statement, params) + + def _skip_applied(self, idempotent_condition: Condition) -> bool: + if not self.IDEMPOTENT_SQL: + # in case of failure of creating indexes concurrently index will be created but will be invalid + # for this case reindex statement added to recreate valid index in IDEMPOTENT_SQL mode + # but if IDEMPOTENT_SQL mode is disabled we need to skip this extra reindex sql + return idempotent_condition is not None and idempotent_condition.idempotent_mode_only + + if idempotent_condition is None: + return False + + with self.connection.cursor() as cursor: + cursor.execute(idempotent_condition.sql) + exists = cursor.fetchone() is not None + if idempotent_condition.exists: + return not exists + return exists @contextmanager def _set_operation_timeout(self, statement_timeout=None, lock_timeout=None): diff --git a/tests/apps/idempotency_add_check_app/__init__.py b/tests/apps/idempotency_add_check_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_check_app/migrations/0001_initial.py b/tests/apps/idempotency_add_check_app/migrations/0001_initial.py new file mode 100644 index 0000000..ce33651 --- /dev/null +++ b/tests/apps/idempotency_add_check_app/migrations/0001_initial.py @@ -0,0 +1,42 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="RelatedTestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField(null=True)), + ], + ), + migrations.CreateModel( + name="TestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField()), + ("test_field_str", models.CharField(max_length=10)), + ], + ), + ] diff --git a/tests/apps/idempotency_add_check_app/migrations/0002_relatedtesttable_idempotency_add_check_app_relatedtesttable_check.py b/tests/apps/idempotency_add_check_app/migrations/0002_relatedtesttable_idempotency_add_check_app_relatedtesttable_check.py new file mode 100644 index 0000000..c0d487b --- /dev/null +++ b/tests/apps/idempotency_add_check_app/migrations/0002_relatedtesttable_idempotency_add_check_app_relatedtesttable_check.py @@ -0,0 +1,20 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("idempotency_add_check_app", "0001_initial"), + ] + + operations = [ + migrations.AddConstraint( + model_name="relatedtesttable", + constraint=models.CheckConstraint( + check=models.Q(("test_field_int__gt", 0)), + name="idempotency_add_check_app_relatedtesttable_check", + ), + ), + ] diff --git a/tests/apps/idempotency_add_check_app/migrations/__init__.py b/tests/apps/idempotency_add_check_app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_check_app/models.py b/tests/apps/idempotency_add_check_app/models.py new file mode 100644 index 0000000..063b87e --- /dev/null +++ b/tests/apps/idempotency_add_check_app/models.py @@ -0,0 +1,18 @@ +from django.db import models + + +class TestTable(models.Model): + test_field_int = models.IntegerField() + test_field_str = models.CharField(max_length=10) + + +class RelatedTestTable(models.Model): + test_field_int = models.IntegerField(null=True) + + class Meta: + constraints = [ + models.CheckConstraint( + check=models.Q(test_field_int__gt=0), + name="idempotency_add_check_app_relatedtesttable_check", + ) + ] diff --git a/tests/apps/idempotency_add_column_app/__init__.py b/tests/apps/idempotency_add_column_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_column_app/migrations/0001_initial.py b/tests/apps/idempotency_add_column_app/migrations/0001_initial.py new file mode 100644 index 0000000..ce33651 --- /dev/null +++ b/tests/apps/idempotency_add_column_app/migrations/0001_initial.py @@ -0,0 +1,42 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="RelatedTestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField(null=True)), + ], + ), + migrations.CreateModel( + name="TestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField()), + ("test_field_str", models.CharField(max_length=10)), + ], + ), + ] diff --git a/tests/apps/idempotency_add_column_app/migrations/0002_relatedtesttable_test_model.py b/tests/apps/idempotency_add_column_app/migrations/0002_relatedtesttable_test_model.py new file mode 100644 index 0000000..cfda7f9 --- /dev/null +++ b/tests/apps/idempotency_add_column_app/migrations/0002_relatedtesttable_test_model.py @@ -0,0 +1,25 @@ +# Generated by Django 5.0 on 2024-04-22 17:47 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("idempotency_add_column_app", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="relatedtesttable", + name="test_model", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="idempotency_add_column_app.testtable", + ), + ), + ] diff --git a/tests/apps/idempotency_add_column_app/migrations/__init__.py b/tests/apps/idempotency_add_column_app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_column_app/models.py b/tests/apps/idempotency_add_column_app/models.py new file mode 100644 index 0000000..5f585b1 --- /dev/null +++ b/tests/apps/idempotency_add_column_app/models.py @@ -0,0 +1,11 @@ +from django.db import models + + +class TestTable(models.Model): + test_field_int = models.IntegerField() + test_field_str = models.CharField(max_length=10) + + +class RelatedTestTable(models.Model): + test_field_int = models.IntegerField(null=True) + test_model = models.OneToOneField(TestTable, null=True, on_delete=models.CASCADE) diff --git a/tests/apps/idempotency_add_foreign_key_app/__init__.py b/tests/apps/idempotency_add_foreign_key_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_foreign_key_app/migrations/0001_initial.py b/tests/apps/idempotency_add_foreign_key_app/migrations/0001_initial.py new file mode 100644 index 0000000..ce33651 --- /dev/null +++ b/tests/apps/idempotency_add_foreign_key_app/migrations/0001_initial.py @@ -0,0 +1,42 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="RelatedTestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField(null=True)), + ], + ), + migrations.CreateModel( + name="TestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField()), + ("test_field_str", models.CharField(max_length=10)), + ], + ), + ] diff --git a/tests/apps/idempotency_add_foreign_key_app/migrations/0002_alter_relatedtesttable_test_field_int.py b/tests/apps/idempotency_add_foreign_key_app/migrations/0002_alter_relatedtesttable_test_field_int.py new file mode 100644 index 0000000..560ea3b --- /dev/null +++ b/tests/apps/idempotency_add_foreign_key_app/migrations/0002_alter_relatedtesttable_test_field_int.py @@ -0,0 +1,26 @@ +# Generated by Django 5.0 on 2024-04-23 10:59 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("idempotency_add_foreign_key_app", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="relatedtesttable", + name="test_field_int", + field=models.OneToOneField( + db_column="test_field_int", + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="idempotency_add_foreign_key_app.testtable", + ), + ), + ] diff --git a/tests/apps/idempotency_add_foreign_key_app/migrations/__init__.py b/tests/apps/idempotency_add_foreign_key_app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_foreign_key_app/models.py b/tests/apps/idempotency_add_foreign_key_app/models.py new file mode 100644 index 0000000..a409d94 --- /dev/null +++ b/tests/apps/idempotency_add_foreign_key_app/models.py @@ -0,0 +1,15 @@ +from django.db import models + + +class TestTable(models.Model): + test_field_int = models.IntegerField() + test_field_str = models.CharField(max_length=10) + + +class RelatedTestTable(models.Model): + test_field_int = models.OneToOneField( + TestTable, + null=True, + on_delete=models.CASCADE, + db_column="test_field_int", + ) diff --git a/tests/apps/idempotency_add_primary_key_app/__init__.py b/tests/apps/idempotency_add_primary_key_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_primary_key_app/migrations/0001_initial.py b/tests/apps/idempotency_add_primary_key_app/migrations/0001_initial.py new file mode 100644 index 0000000..00b0c3e --- /dev/null +++ b/tests/apps/idempotency_add_primary_key_app/migrations/0001_initial.py @@ -0,0 +1,42 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="RelatedTestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField()), + ], + ), + migrations.CreateModel( + name="TestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField()), + ("test_field_str", models.CharField(max_length=10)), + ], + ), + ] diff --git a/tests/apps/idempotency_add_primary_key_app/migrations/0002_remove_relatedtesttable_id_and_more.py b/tests/apps/idempotency_add_primary_key_app/migrations/0002_remove_relatedtesttable_id_and_more.py new file mode 100644 index 0000000..9aa2e01 --- /dev/null +++ b/tests/apps/idempotency_add_primary_key_app/migrations/0002_remove_relatedtesttable_id_and_more.py @@ -0,0 +1,22 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("idempotency_add_primary_key_app", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="relatedtesttable", + name="id", + ), + migrations.AlterField( + model_name="relatedtesttable", + name="test_field_int", + field=models.IntegerField(primary_key=True, serialize=False), + ), + ] diff --git a/tests/apps/idempotency_add_primary_key_app/migrations/__init__.py b/tests/apps/idempotency_add_primary_key_app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_primary_key_app/models.py b/tests/apps/idempotency_add_primary_key_app/models.py new file mode 100644 index 0000000..43295a7 --- /dev/null +++ b/tests/apps/idempotency_add_primary_key_app/models.py @@ -0,0 +1,10 @@ +from django.db import models + + +class TestTable(models.Model): + test_field_int = models.IntegerField() + test_field_str = models.CharField(max_length=10) + + +class RelatedTestTable(models.Model): + test_field_int = models.IntegerField(primary_key=True) diff --git a/tests/apps/idempotency_add_unique_app/__init__.py b/tests/apps/idempotency_add_unique_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_unique_app/migrations/0001_initial.py b/tests/apps/idempotency_add_unique_app/migrations/0001_initial.py new file mode 100644 index 0000000..ce33651 --- /dev/null +++ b/tests/apps/idempotency_add_unique_app/migrations/0001_initial.py @@ -0,0 +1,42 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="RelatedTestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField(null=True)), + ], + ), + migrations.CreateModel( + name="TestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField()), + ("test_field_str", models.CharField(max_length=10)), + ], + ), + ] diff --git a/tests/apps/idempotency_add_unique_app/migrations/0002_alter_relatedtesttable_test_field_int.py b/tests/apps/idempotency_add_unique_app/migrations/0002_alter_relatedtesttable_test_field_int.py new file mode 100644 index 0000000..aa57824 --- /dev/null +++ b/tests/apps/idempotency_add_unique_app/migrations/0002_alter_relatedtesttable_test_field_int.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("idempotency_add_unique_app", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="relatedtesttable", + name="test_field_int", + field=models.IntegerField(null=True, unique=True), + ), + ] diff --git a/tests/apps/idempotency_add_unique_app/migrations/__init__.py b/tests/apps/idempotency_add_unique_app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_add_unique_app/models.py b/tests/apps/idempotency_add_unique_app/models.py new file mode 100644 index 0000000..996513b --- /dev/null +++ b/tests/apps/idempotency_add_unique_app/models.py @@ -0,0 +1,10 @@ +from django.db import models + + +class TestTable(models.Model): + test_field_int = models.IntegerField() + test_field_str = models.CharField(max_length=10) + + +class RelatedTestTable(models.Model): + test_field_int = models.IntegerField(null=True, unique=True) diff --git a/tests/apps/idempotency_create_table_app/__init__.py b/tests/apps/idempotency_create_table_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_create_table_app/migrations/0001_initial.py b/tests/apps/idempotency_create_table_app/migrations/0001_initial.py new file mode 100644 index 0000000..0251ce5 --- /dev/null +++ b/tests/apps/idempotency_create_table_app/migrations/0001_initial.py @@ -0,0 +1,26 @@ +from django.db import migrations, models + + +def insert_objects(apps, schema_editor): + db_alias = schema_editor.connection.alias + TestTable = apps.get_model("idempotency_create_table_app", "TestTable") + TestTable.objects.using(db_alias).create(test_field_int=1) + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="TestTable", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("test_field_int", models.IntegerField()), + ("test_field_str", models.CharField(max_length=10)), + ], + ), + migrations.RunPython(insert_objects, migrations.RunPython.noop), + ] diff --git a/tests/apps/idempotency_create_table_app/migrations/0002_relatedtesttable_and_more.py b/tests/apps/idempotency_create_table_app/migrations/0002_relatedtesttable_and_more.py new file mode 100644 index 0000000..71efa32 --- /dev/null +++ b/tests/apps/idempotency_create_table_app/migrations/0002_relatedtesttable_and_more.py @@ -0,0 +1,46 @@ +# Generated by Django 5.0 on 2024-04-22 15:54 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("idempotency_create_table_app", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="RelatedTestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField(null=True)), + ( + "test_model", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="idempotency_create_table_app.testtable", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="relatedtesttable", + constraint=models.UniqueConstraint( + fields=("test_model", "test_field_int"), + name="idempotency_create_table_app_relatedtesttable_uniq", + ), + ), + ] diff --git a/tests/apps/idempotency_create_table_app/migrations/__init__.py b/tests/apps/idempotency_create_table_app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_create_table_app/models.py b/tests/apps/idempotency_create_table_app/models.py new file mode 100644 index 0000000..28b0067 --- /dev/null +++ b/tests/apps/idempotency_create_table_app/models.py @@ -0,0 +1,19 @@ +from django.db import models + + +class TestTable(models.Model): + test_field_int = models.IntegerField() + test_field_str = models.CharField(max_length=10) + + +class RelatedTestTable(models.Model): + test_model = models.ForeignKey(TestTable, null=True, on_delete=models.CASCADE) + test_field_int = models.IntegerField(null=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + name="idempotency_create_table_app_relatedtesttable_uniq", + fields=["test_model", "test_field_int"], + ) + ] diff --git a/tests/apps/idempotency_set_not_null_app/__init__.py b/tests/apps/idempotency_set_not_null_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_set_not_null_app/migrations/0001_initial.py b/tests/apps/idempotency_set_not_null_app/migrations/0001_initial.py new file mode 100644 index 0000000..ce33651 --- /dev/null +++ b/tests/apps/idempotency_set_not_null_app/migrations/0001_initial.py @@ -0,0 +1,42 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="RelatedTestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField(null=True)), + ], + ), + migrations.CreateModel( + name="TestTable", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("test_field_int", models.IntegerField()), + ("test_field_str", models.CharField(max_length=10)), + ], + ), + ] diff --git a/tests/apps/idempotency_set_not_null_app/migrations/0002_alter_relatedtesttable_test_field_int.py b/tests/apps/idempotency_set_not_null_app/migrations/0002_alter_relatedtesttable_test_field_int.py new file mode 100644 index 0000000..9a24337 --- /dev/null +++ b/tests/apps/idempotency_set_not_null_app/migrations/0002_alter_relatedtesttable_test_field_int.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("idempotency_set_not_null_app", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="relatedtesttable", + name="test_field_int", + field=models.IntegerField(), + ), + ] diff --git a/tests/apps/idempotency_set_not_null_app/migrations/__init__.py b/tests/apps/idempotency_set_not_null_app/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/idempotency_set_not_null_app/models.py b/tests/apps/idempotency_set_not_null_app/models.py new file mode 100644 index 0000000..351a84f --- /dev/null +++ b/tests/apps/idempotency_set_not_null_app/models.py @@ -0,0 +1,10 @@ +from django.db import models + + +class TestTable(models.Model): + test_field_int = models.IntegerField() + test_field_str = models.CharField(max_length=10) + + +class RelatedTestTable(models.Model): + test_field_int = models.IntegerField() diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 441ea07..d8a96fb 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,4 +1,9 @@ +import os +import subprocess + +from django.conf import settings from django.core.management.commands.migrate import Command as MigrateCommand +from django.db import connection def migrate(params=None): @@ -7,3 +12,47 @@ def migrate(params=None): options = parser.parse_args(params or []) cmd_options = vars(options) return command.execute(**cmd_options) + + +def one_line_sql(sql: str) -> str: + return sql.replace(" ", "").replace("\n", " ").replace("( ", "(").replace(" )", ")").replace(" ", " ").strip() + + +def pg_dump(table: str) -> str: + host = settings.DATABASES["default"]["HOST"] + port = settings.DATABASES["default"]["PORT"] + name = settings.DATABASES["default"]["NAME"] + user = settings.DATABASES["default"]["USER"] + password = settings.DATABASES["default"]["PASSWORD"] + env = os.environ.copy() | {"PGPASSWORD": password} + cmd = f"pg_dump -h {host} -p {port} -U {user} -d {name} -s -t {table}" + popen = subprocess.run(cmd, env=env, text=True, shell=True, capture_output=True) + return popen.stdout + + +def is_valid_index(table: str, index: str) -> bool: + with connection.cursor() as cursor: + cursor.execute(""" + SELECT indisvalid + FROM pg_index + WHERE indrelid = %s::regclass::oid + AND indexrelid = %s::regclass::oid + """, [table, index]) + data = cursor.fetchone() + if data is None: + raise ValueError(f"index {index} not found for {table}") + return data[0] + + +def is_valid_constraint(table: str, constraint: str) -> bool: + with connection.cursor() as cursor: + cursor.execute(""" + SELECT convalidated + FROM pg_constraint + WHERE conrelid = %s::regclass::oid + AND conname = %s + """, [table, constraint]) + data = cursor.fetchone() + if data is None: + raise ValueError(f"constraint {constraint} not found for {table}") + return data[0] diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 341fc8c..32aea93 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -1,3 +1,7 @@ +import textwrap + +from django.core.management import call_command +from django.db import connection, IntegrityError from django.test import modify_settings, override_settings import pytest @@ -6,7 +10,7 @@ UnsafeOperationException ) from tests import skip_for_default_django_backend -from tests.integration import migrate +from tests.integration import migrate, pg_dump, is_valid_index, is_valid_constraint, one_line_sql @pytest.mark.django_db(transaction=True) @@ -130,3 +134,672 @@ def test_decimal_to_float_app(): # backward migrate(['decimal_to_float_app', 'zero']) + + +@pytest.mark.django_db(transaction=True) +@modify_settings(INSTALLED_APPS={"append": "tests.apps.idempotency_create_table_app"}) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=True) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=True) +def test_idempotency_create_table(): + _create_table_sql = one_line_sql(""" + CREATE TABLE "idempotency_create_table_app_relatedtesttable" ( + "id" integer NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + "test_field_int" integer NULL, + "test_model_id" integer NULL + ); + """) + _create_unique_index_sql = one_line_sql(""" + CREATE UNIQUE INDEX CONCURRENTLY "idempotency_create_table_app_relatedtesttable_uniq" + ON "idempotency_create_table_app_relatedtesttable" ("test_model_id", "test_field_int"); + """) + _create_unique_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_create_table_app_relatedtesttable" + ADD CONSTRAINT "idempotency_create_table_app_relatedtesttable_uniq" + UNIQUE USING INDEX "idempotency_create_table_app_relatedtesttable_uniq"; + """) + _create_foreign_key_sql = one_line_sql(""" + ALTER TABLE "idempotency_create_table_app_relatedtesttable" + ADD CONSTRAINT "idempotency_create_t_test_model_id_09b52f79_fk_idempoten" + FOREIGN KEY ("test_model_id") + REFERENCES "idempotency_create_table_app_testtable" ("id") + DEFERRABLE INITIALLY DEFERRED NOT VALID; + """) + _validate_foreign_key_sql = one_line_sql(""" + ALTER TABLE "idempotency_create_table_app_relatedtesttable" + VALIDATE CONSTRAINT "idempotency_create_t_test_model_id_09b52f79_fk_idempoten"; + """) + _create_index_sql = one_line_sql(""" + CREATE INDEX CONCURRENTLY "idempotency_create_table_a_test_model_id_09b52f79" + ON "idempotency_create_table_app_relatedtesttable" ("test_model_id"); + """) + + call_command("migrate", "idempotency_create_table_app", "0001") + with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False): + migration_sql = call_command("sqlmigrate", "idempotency_create_table_app", "0002") + assert migration_sql == textwrap.dedent(f""" + -- + -- Create model RelatedTestTable + -- + {_create_table_sql} + -- + -- Create constraint idempotency_create_table_app_relatedtesttable_uniq on model relatedtesttable + -- + {_create_unique_index_sql} + {_create_unique_constraint_sql} + {_create_foreign_key_sql} + {_validate_foreign_key_sql} + {_create_index_sql} + """).strip() + + # get target schema + call_command("migrate", "idempotency_create_table_app") + schema = pg_dump("idempotency_create_table_app_relatedtesttable") + + # case 1 + call_command("migrate", "idempotency_create_table_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_table_sql) + call_command("migrate", "idempotency_create_table_app") + assert pg_dump("idempotency_create_table_app_relatedtesttable") == schema + + # case 2.1 + call_command("migrate", "idempotency_create_table_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_table_sql) + cursor.execute(_create_unique_index_sql) + call_command("migrate", "idempotency_create_table_app") + assert pg_dump("idempotency_create_table_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_create_table_app_relatedtesttable", + "idempotency_create_table_app_relatedtesttable_uniq", + ) + + # case 2.2 + call_command("migrate", "idempotency_create_table_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_table_sql) + cursor.execute(""" + -- ensure index invalid + INSERT INTO idempotency_create_table_app_relatedtesttable + (id, test_field_int, test_model_id) VALUES (1, 1, 1), (2, 1, 1) + """) + with pytest.raises(IntegrityError): + cursor.execute(_create_unique_index_sql) + cursor.execute(""" + DELETE FROM idempotency_create_table_app_relatedtesttable + """) + assert not is_valid_index( + "idempotency_create_table_app_relatedtesttable", + "idempotency_create_table_app_relatedtesttable_uniq", + ) + call_command("migrate", "idempotency_create_table_app") + assert pg_dump("idempotency_create_table_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_create_table_app_relatedtesttable", + "idempotency_create_table_app_relatedtesttable_uniq", + ) + + # case 3 + call_command("migrate", "idempotency_create_table_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_table_sql) + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + call_command("migrate", "idempotency_create_table_app") + assert pg_dump("idempotency_create_table_app_relatedtesttable") == schema + + # case 4 + call_command("migrate", "idempotency_create_table_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_table_sql) + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + cursor.execute(_create_foreign_key_sql) + call_command("migrate", "idempotency_create_table_app") + assert pg_dump("idempotency_create_table_app_relatedtesttable") == schema + + # case 5 + call_command("migrate", "idempotency_create_table_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_table_sql) + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + cursor.execute(_create_foreign_key_sql) + cursor.execute(_validate_foreign_key_sql) + call_command("migrate", "idempotency_create_table_app") + assert pg_dump("idempotency_create_table_app_relatedtesttable") == schema + + # case 6 + call_command("migrate", "idempotency_create_table_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_table_sql) + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + cursor.execute(_create_foreign_key_sql) + cursor.execute(_validate_foreign_key_sql) + cursor.execute(_create_index_sql) + call_command("migrate", "idempotency_create_table_app") + assert pg_dump("idempotency_create_table_app_relatedtesttable") == schema + + +@pytest.mark.django_db(transaction=True) +@modify_settings(INSTALLED_APPS={"append": "tests.apps.idempotency_add_column_app"}) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=True) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=True) +def test_idempotency_add_column(): + _add_column_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_column_app_relatedtesttable" + ADD COLUMN "test_model_id" integer NULL; + """) + _create_unique_index_sql = one_line_sql(""" + CREATE UNIQUE INDEX CONCURRENTLY "idempotency_add_column_a_test_model_id_76f0c43b_uniq" + ON "idempotency_add_column_app_relatedtesttable" ("test_model_id"); + """) + _create_unique_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_column_app_relatedtesttable" + ADD CONSTRAINT "idempotency_add_column_a_test_model_id_76f0c43b_uniq" + UNIQUE USING INDEX "idempotency_add_column_a_test_model_id_76f0c43b_uniq"; + """) + _create_foreign_key_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_column_app_relatedtesttable" + ADD CONSTRAINT "idempotency_add_colu_test_model_id_76f0c43b_fk_idempoten" + FOREIGN KEY ("test_model_id") + REFERENCES "idempotency_add_column_app_testtable" ("id") + DEFERRABLE INITIALLY DEFERRED NOT VALID; + """) + _validate_foreign_key_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_column_app_relatedtesttable" + VALIDATE CONSTRAINT "idempotency_add_colu_test_model_id_76f0c43b_fk_idempoten"; + """) + + call_command("migrate", "idempotency_add_column_app", "0001") + with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False): + migration_sql = call_command("sqlmigrate", "idempotency_add_column_app", "0002") + assert migration_sql == textwrap.dedent(f""" + -- + -- Add field test_model to relatedtesttable + -- + {_add_column_sql} + {_create_unique_index_sql} + {_create_unique_constraint_sql} + {_create_foreign_key_constraint_sql} + {_validate_foreign_key_constraint_sql} + """).strip() + + # get target schema + call_command("migrate", "idempotency_add_column_app") + schema = pg_dump("idempotency_add_column_app_relatedtesttable") + + # case 1 + call_command("migrate", "idempotency_add_column_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_add_column_sql) + call_command("migrate", "idempotency_add_column_app") + assert pg_dump("idempotency_add_column_app_relatedtesttable") == schema + + # case 2.1 + call_command("migrate", "idempotency_add_column_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_add_column_sql) + cursor.execute(_create_unique_index_sql) + call_command("migrate", "idempotency_add_column_app") + assert pg_dump("idempotency_add_column_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_add_column_app_relatedtesttable", + "idempotency_add_column_a_test_model_id_76f0c43b_uniq", + ) + + # case 2.2 + call_command("migrate", "idempotency_add_column_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_add_column_sql) + cursor.execute(""" + -- ensure index invalid + INSERT INTO idempotency_add_column_app_relatedtesttable + (id, test_field_int, test_model_id) VALUES (1, 1, 1), (2, 1, 1) + """) + with pytest.raises(IntegrityError): + cursor.execute(_create_unique_index_sql) + cursor.execute(""" + DELETE FROM idempotency_add_column_app_relatedtesttable + """) + assert not is_valid_index( + "idempotency_add_column_app_relatedtesttable", + "idempotency_add_column_a_test_model_id_76f0c43b_uniq", + ) + call_command("migrate", "idempotency_add_column_app") + assert pg_dump("idempotency_add_column_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_add_column_app_relatedtesttable", + "idempotency_add_column_a_test_model_id_76f0c43b_uniq", + ) + + # case 3 + call_command("migrate", "idempotency_add_column_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_add_column_sql) + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + call_command("migrate", "idempotency_add_column_app") + assert pg_dump("idempotency_add_column_app_relatedtesttable") == schema + + # case 4 + call_command("migrate", "idempotency_add_column_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_add_column_sql) + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + cursor.execute(_create_foreign_key_constraint_sql) + call_command("migrate", "idempotency_add_column_app") + assert pg_dump("idempotency_add_column_app_relatedtesttable") == schema + + # case 5 + call_command("migrate", "idempotency_add_column_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_add_column_sql) + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + cursor.execute(_create_foreign_key_constraint_sql) + cursor.execute(_validate_foreign_key_constraint_sql) + call_command("migrate", "idempotency_add_column_app") + assert pg_dump("idempotency_add_column_app_relatedtesttable") == schema + + +@pytest.mark.django_db(transaction=True) +@modify_settings(INSTALLED_APPS={"append": "tests.apps.idempotency_set_not_null_app"}) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=True) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=True) +def test_idempotency_set_not_null(): + _create_check_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_set_not_null_app_relatedtesttable" + ADD CONSTRAINT "idempotency_set_not_nu_test_field_int_76dfbad6_notnull" + CHECK ("test_field_int" IS NOT NULL) NOT VALID; + """) + _validate_check_constraint_sql = one_line_sql( + """ + ALTER TABLE "idempotency_set_not_null_app_relatedtesttable" + VALIDATE CONSTRAINT "idempotency_set_not_nu_test_field_int_76dfbad6_notnull"; + """) + _set_column_not_null_sql = one_line_sql(""" + ALTER TABLE "idempotency_set_not_null_app_relatedtesttable" + ALTER COLUMN "test_field_int" SET NOT NULL; + """) + _drop_check_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_set_not_null_app_relatedtesttable" + DROP CONSTRAINT "idempotency_set_not_nu_test_field_int_76dfbad6_notnull"; + """) + + call_command("migrate", "idempotency_set_not_null_app", "0001") + with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False): + migration_sql = call_command("sqlmigrate", "idempotency_set_not_null_app", "0002") + assert migration_sql == textwrap.dedent(f""" + -- + -- Alter field test_field_int on relatedtesttable + -- + -- (no-op) + {_create_check_constraint_sql} + {_validate_check_constraint_sql} + {_set_column_not_null_sql} + {_drop_check_constraint_sql} + """).strip() + + # get target schema + call_command("migrate", "idempotency_set_not_null_app") + schema = pg_dump("idempotency_set_not_null_app_relatedtesttable") + + # case 1 + call_command("migrate", "idempotency_set_not_null_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_check_constraint_sql) + call_command("migrate", "idempotency_set_not_null_app") + assert pg_dump("idempotency_set_not_null_app_relatedtesttable") == schema + + # case 2 + call_command("migrate", "idempotency_set_not_null_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_check_constraint_sql) + cursor.execute(_validate_check_constraint_sql) + call_command("migrate", "idempotency_set_not_null_app") + assert pg_dump("idempotency_set_not_null_app_relatedtesttable") == schema + + # case 3 + call_command("migrate", "idempotency_set_not_null_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_check_constraint_sql) + cursor.execute(_validate_check_constraint_sql) + cursor.execute(_set_column_not_null_sql) + call_command("migrate", "idempotency_set_not_null_app") + assert pg_dump("idempotency_set_not_null_app_relatedtesttable") == schema + + # case 4 + call_command("migrate", "idempotency_set_not_null_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_check_constraint_sql) + cursor.execute(_validate_check_constraint_sql) + cursor.execute(_set_column_not_null_sql) + cursor.execute(_drop_check_constraint_sql) + call_command("migrate", "idempotency_set_not_null_app") + assert pg_dump("idempotency_set_not_null_app_relatedtesttable") == schema + + +@pytest.mark.django_db(transaction=True) +@modify_settings(INSTALLED_APPS={"append": "tests.apps.idempotency_add_check_app"}) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=True) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=True) +def test_idempotency_add_check(): + _create_check_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_check_app_relatedtesttable" + ADD CONSTRAINT "idempotency_add_check_app_relatedtesttable_check" + CHECK ("test_field_int" > 0) NOT VALID; + """) + _validate_check_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_check_app_relatedtesttable" + VALIDATE CONSTRAINT "idempotency_add_check_app_relatedtesttable_check"; + """) + + call_command("migrate", "idempotency_add_check_app", "0001") + with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False): + migration_sql = call_command("sqlmigrate", "idempotency_add_check_app", "0002") + assert migration_sql == textwrap.dedent(f""" + -- + -- Create constraint idempotency_add_check_app_relatedtesttable_check on model relatedtesttable + -- + {_create_check_constraint_sql} + {_validate_check_constraint_sql} + """).strip() + + # get target schema + call_command("migrate", "idempotency_add_check_app") + schema = pg_dump("idempotency_add_check_app_relatedtesttable") + + # case 1 + call_command("migrate", "idempotency_add_check_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_check_constraint_sql) + call_command("migrate", "idempotency_add_check_app") + assert pg_dump("idempotency_add_check_app_relatedtesttable") == schema + + # case 2 + call_command("migrate", "idempotency_add_check_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_check_constraint_sql) + cursor.execute(_validate_check_constraint_sql) + call_command("migrate", "idempotency_add_check_app") + assert pg_dump("idempotency_add_check_app_relatedtesttable") == schema + + +@pytest.mark.django_db(transaction=True) +@modify_settings(INSTALLED_APPS={"append": "tests.apps.idempotency_add_foreign_key_app"}) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=True) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=True) +def test_idempotency_add_foreign_key(): + _create_unique_index_sql = one_line_sql(""" + CREATE UNIQUE INDEX CONCURRENTLY "idempotency_add_foreign__test_field_int_fa01ee40_uniq" + ON "idempotency_add_foreign_key_app_relatedtesttable" ("test_field_int"); + """) + _create_unique_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_foreign_key_app_relatedtesttable" + ADD CONSTRAINT "idempotency_add_foreign__test_field_int_fa01ee40_uniq" + UNIQUE USING INDEX "idempotency_add_foreign__test_field_int_fa01ee40_uniq"; + """) + _create_foreign_key_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_foreign_key_app_relatedtesttable" + ADD CONSTRAINT "idempotency_add_fore_test_field_int_fa01ee40_fk_idempoten" + FOREIGN KEY ("test_field_int") + REFERENCES "idempotency_add_foreign_key_app_testtable" ("id") + DEFERRABLE INITIALLY DEFERRED NOT VALID; + """) + _validate_foreign_key_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_foreign_key_app_relatedtesttable" + VALIDATE CONSTRAINT "idempotency_add_fore_test_field_int_fa01ee40_fk_idempoten"; + """) + + call_command("migrate", "idempotency_add_foreign_key_app", "0001") + with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False): + migration_sql = call_command("sqlmigrate", "idempotency_add_foreign_key_app", "0002") + assert migration_sql == textwrap.dedent(f""" + -- + -- Alter field test_field_int on relatedtesttable + -- + {_create_unique_index_sql} + {_create_unique_constraint_sql} + {_create_foreign_key_constraint_sql} + {_validate_foreign_key_constraint_sql} + """).strip() + + # get target schema + call_command("migrate", "idempotency_add_foreign_key_app") + schema = pg_dump("idempotency_add_foreign_key_app_relatedtesttable") + + # case 1.1 + call_command("migrate", "idempotency_add_foreign_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_unique_index_sql) + call_command("migrate", "idempotency_add_foreign_key_app") + assert pg_dump("idempotency_add_foreign_key_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_add_foreign_key_app_relatedtesttable", + "idempotency_add_foreign__test_field_int_fa01ee40_uniq", + ) + + # case 1.2 + call_command("migrate", "idempotency_add_foreign_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(""" + -- ensure index invalid + INSERT INTO idempotency_add_foreign_key_app_relatedtesttable + (id, test_field_int) VALUES (1, 1), (2, 1) + """) + with pytest.raises(IntegrityError): + cursor.execute(_create_unique_index_sql) + cursor.execute(""" + DELETE FROM idempotency_add_foreign_key_app_relatedtesttable + """) + assert not is_valid_index( + "idempotency_add_foreign_key_app_relatedtesttable", + "idempotency_add_foreign__test_field_int_fa01ee40_uniq", + ) + call_command("migrate", "idempotency_add_foreign_key_app") + assert pg_dump("idempotency_add_foreign_key_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_add_foreign_key_app_relatedtesttable", + "idempotency_add_foreign__test_field_int_fa01ee40_uniq", + ) + + # case 2 + call_command("migrate", "idempotency_add_foreign_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + call_command("migrate", "idempotency_add_foreign_key_app") + assert pg_dump("idempotency_add_foreign_key_app_relatedtesttable") == schema + + # case 3 + call_command("migrate", "idempotency_add_foreign_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + cursor.execute(_create_foreign_key_constraint_sql) + call_command("migrate", "idempotency_add_foreign_key_app") + assert pg_dump("idempotency_add_foreign_key_app_relatedtesttable") == schema + + # case 4 + call_command("migrate", "idempotency_add_foreign_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + cursor.execute(_create_foreign_key_constraint_sql) + cursor.execute(_validate_foreign_key_constraint_sql) + call_command("migrate", "idempotency_add_foreign_key_app") + assert pg_dump("idempotency_add_foreign_key_app_relatedtesttable") == schema + + +@pytest.mark.django_db(transaction=True) +@modify_settings(INSTALLED_APPS={"append": "tests.apps.idempotency_add_unique_app"}) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=True) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=True) +def test_idempotency_add_unique(): + _create_unique_index_sql = one_line_sql(""" + CREATE UNIQUE INDEX CONCURRENTLY "idempotency_add_unique_a_test_field_int_01c4f0c0_uniq" + ON "idempotency_add_unique_app_relatedtesttable" ("test_field_int"); + """) + _create_unique_constraint_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_unique_app_relatedtesttable" + ADD CONSTRAINT "idempotency_add_unique_a_test_field_int_01c4f0c0_uniq" + UNIQUE USING INDEX "idempotency_add_unique_a_test_field_int_01c4f0c0_uniq"; + """) + + call_command("migrate", "idempotency_add_unique_app", "0001") + with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False): + migration_sql = call_command("sqlmigrate", "idempotency_add_unique_app", "0002") + assert migration_sql == textwrap.dedent(f""" + -- + -- Alter field test_field_int on relatedtesttable + -- + {_create_unique_index_sql} + {_create_unique_constraint_sql} + """).strip() + + # get target schema + call_command("migrate", "idempotency_add_unique_app") + schema = pg_dump("idempotency_add_unique_app_relatedtesttable") + + # case 1.1 + call_command("migrate", "idempotency_add_unique_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_unique_index_sql) + call_command("migrate", "idempotency_add_unique_app") + assert pg_dump("idempotency_add_unique_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_add_unique_app_relatedtesttable", + "idempotency_add_unique_a_test_field_int_01c4f0c0_uniq", + ) + + # case 1.2 + call_command("migrate", "idempotency_add_unique_app", "0001") + with connection.cursor() as cursor: + cursor.execute(""" + -- ensure index invalid + INSERT INTO idempotency_add_unique_app_relatedtesttable + (id, test_field_int) VALUES (1, 1), (2, 1) + """) + with pytest.raises(IntegrityError): + cursor.execute(_create_unique_index_sql) + cursor.execute(""" + DELETE FROM idempotency_add_unique_app_relatedtesttable + """) + assert not is_valid_index( + "idempotency_add_unique_app_relatedtesttable", + "idempotency_add_unique_a_test_field_int_01c4f0c0_uniq", + ) + call_command("migrate", "idempotency_add_unique_app") + assert pg_dump("idempotency_add_unique_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_add_unique_app_relatedtesttable", + "idempotency_add_unique_a_test_field_int_01c4f0c0_uniq", + ) + + # case 2 + call_command("migrate", "idempotency_add_unique_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_unique_constraint_sql) + call_command("migrate", "idempotency_add_unique_app") + assert pg_dump("idempotency_add_unique_app_relatedtesttable") == schema + + +@pytest.mark.django_db(transaction=True) +@modify_settings(INSTALLED_APPS={"append": "tests.apps.idempotency_add_primary_key_app"}) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=True) +@override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=True) +def test_idempotency_add_primary_key(): + _drop_column_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_primary_key_app_relatedtesttable" + DROP COLUMN "id" CASCADE; + """) + _create_unique_index_sql = one_line_sql( + """ + CREATE UNIQUE INDEX CONCURRENTLY "idempotency_add_primary_k_test_field_int_e9cebf24_pk" + ON "idempotency_add_primary_key_app_relatedtesttable" ("test_field_int"); + """) + _create_primary_key_sql = one_line_sql(""" + ALTER TABLE "idempotency_add_primary_key_app_relatedtesttable" + ADD CONSTRAINT "idempotency_add_primary_k_test_field_int_e9cebf24_pk" + PRIMARY KEY USING INDEX "idempotency_add_primary_k_test_field_int_e9cebf24_pk"; + """) + + call_command("migrate", "idempotency_add_primary_key_app", "0001") + with override_settings(ZERO_DOWNTIME_MIGRATIONS_IDEMPOTENT_SQL=False): + migration_sql = call_command("sqlmigrate", "idempotency_add_primary_key_app", "0002") + assert migration_sql == textwrap.dedent(f""" + -- + -- Remove field id from relatedtesttable + -- + {_drop_column_sql} + -- + -- Alter field test_field_int on relatedtesttable + -- + {_create_unique_index_sql} + {_create_primary_key_sql} + """).strip() + + # get target schema + call_command("migrate", "idempotency_add_primary_key_app") + schema = pg_dump("idempotency_add_primary_key_app_relatedtesttable") + + # case 1 + with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): + call_command("migrate", "idempotency_add_primary_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_drop_column_sql) + call_command("migrate", "idempotency_add_primary_key_app") + assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == schema + + # case 2.1 + with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): + call_command("migrate", "idempotency_add_primary_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_drop_column_sql) + cursor.execute(_create_unique_index_sql) + call_command("migrate", "idempotency_add_primary_key_app") + assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_add_primary_key_app_relatedtesttable", + "idempotency_add_primary_k_test_field_int_e9cebf24_pk", + ) + + # case 2.2 + with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): + call_command("migrate", "idempotency_add_primary_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_drop_column_sql) + cursor.execute(""" + -- ensure index invalid + INSERT INTO idempotency_add_primary_key_app_relatedtesttable + (test_field_int) VALUES (1), (1) + """) + with pytest.raises(IntegrityError): + cursor.execute(_create_unique_index_sql) + cursor.execute(""" + DELETE FROM idempotency_add_primary_key_app_relatedtesttable + """) + assert not is_valid_index( + "idempotency_add_primary_key_app_relatedtesttable", + "idempotency_add_primary_k_test_field_int_e9cebf24_pk", + ) + call_command("migrate", "idempotency_add_primary_key_app") + assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == schema + assert is_valid_constraint( + "idempotency_add_primary_key_app_relatedtesttable", + "idempotency_add_primary_k_test_field_int_e9cebf24_pk", + ) + + # case 3 + with override_settings(ZERO_DOWNTIME_MIGRATIONS_RAISE_FOR_UNSAFE=False): + call_command("migrate", "idempotency_add_primary_key_app", "0001") + with connection.cursor() as cursor: + cursor.execute(_drop_column_sql) + cursor.execute(_create_unique_index_sql) + cursor.execute(_create_primary_key_sql) + call_command("migrate", "idempotency_add_primary_key_app") + assert pg_dump("idempotency_add_primary_key_app_relatedtesttable") == schema diff --git a/tests/settings_make_migrations.py b/tests/settings_make_migrations.py index 0cfdcad..ae7aadb 100644 --- a/tests/settings_make_migrations.py +++ b/tests/settings_make_migrations.py @@ -1,7 +1,7 @@ from .settings import * # noqa: F401, F403 INSTALLED_APPS += [ # noqa: F405 - 'tests.apps.good_flow_alter_table_with_same_name_db_table', + 'tests.apps.good_flow_alter_table_with_same_db_table', 'tests.apps.good_flow_app', 'tests.apps.good_flow_app_concurrently', 'tests.apps.bad_rollback_flow_drop_column_with_notnull_default_app', @@ -11,5 +11,12 @@ 'tests.apps.bad_flow_add_column_with_notnull_default_app', 'tests.apps.bad_flow_add_column_with_notnull_app', 'tests.apps.bad_flow_change_char_type_that_unsafe_app', + 'tests.apps.idempotency_create_table_app', + 'tests.apps.idempotency_add_column_app', + 'tests.apps.idempotency_set_not_null_app', + 'tests.apps.idempotency_add_check_app', + 'tests.apps.idempotency_add_foreign_key_app', + 'tests.apps.idempotency_add_unique_app', + 'tests.apps.idempotency_add_primary_key_app', 'tests.apps.old_notnull_check_constraint_migration_app', ]