Skip to content

Commit

Permalink
🔧 MAINTAIN: Add NotImplementedError for migration downgrades
Browse files Browse the repository at this point in the history
Downgrading storage version is not explicitly supported in aiida-core,
or exposed for the user.
Previous to #5330, some downgrade functionality was required for testing,
since migration tests involved migrating down the global profile,
then migrating it back up to the target version.
These migrations though were often incomplete,
migrating database schema but not the actual data.

Now, migration tests are performed by starting with a separate, empty profile,
and migrating up.
Since these downgrades are no longer required and are un-tested,
we replace their content with an explicit `NotImplementedError`.
  • Loading branch information
chrisjsewell authored and sphuber committed Feb 16, 2022
1 parent 62fbf05 commit 842d47d
Show file tree
Hide file tree
Showing 82 changed files with 106 additions and 778 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -73,35 +73,4 @@ def downgrade():
"""
Downgrade function to the previous schema.
"""
# Create an empty column objname (the data is permanently lost)
op.add_column('db_dblog', sa.Column('objname', sa.VARCHAR(length=255), autoincrement=False, nullable=True))
op.create_index('ix_db_dblog_objname', 'db_dblog', ['objname'])

# Creating a column objpk -
op.add_column('db_dblog', sa.Column('objpk', sa.INTEGER(), autoincrement=False, nullable=True))

# Copy the data back to objpk from dbnode_id
op.execute(text("""UPDATE db_dblog SET objpk=dbnode_id"""))

# Removing the column dbnode_id
op.drop_column('db_dblog', 'dbnode_id')

# Populate objname with correct values
op.execute(
text("""UPDATE db_dblog SET objname=db_dbnode.type
FROM db_dbnode WHERE db_dbnode.id = db_dblog.objpk""")
)

# Enrich metadata with objpk and objname if these keys don't exist
op.execute(
text(
"""UPDATE db_dblog SET metadata = jsonb_set(metadata, '{"objpk"}', to_jsonb(objpk))
WHERE NOT (metadata ?| '{"objpk"}') """
)
)
op.execute(
text(
"""UPDATE db_dblog SET metadata = jsonb_set(metadata, '{"objname"}', to_jsonb(objname))
WHERE NOT (metadata ?| '{"objname"}') """
)
)
raise NotImplementedError('Downgrade of 041a79fc615f.')
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
raise NotImplementedError('Downgrade of 118349c10896.')
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from alembic import op
from sqlalchemy import Integer, String, cast
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.sql import column, func, select, table, text
from sqlalchemy.sql import column, func, select, table

from aiida.backends.sqlalchemy.migrations.utils.utils import load_numpy_array_from_repository

Expand Down Expand Up @@ -69,20 +69,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
connection = op.get_bind()

DbNode = table(
'db_dbnode',
column('id', Integer),
column('uuid', UUID),
column('type', String),
column('attributes', JSONB),
)

nodes = connection.execute(
select(DbNode.c.id,
DbNode.c.uuid).where(DbNode.c.type == op.inline_literal('node.data.array.trajectory.TrajectoryData.'))
).fetchall()

for pk, _ in nodes:
connection.execute(text(f"""UPDATE db_dbnode SET attributes = attributes #- '{{symbols}}' WHERE id = {pk}"""))
raise NotImplementedError('Downgrade of 12536798d4d3.')
Original file line number Diff line number Diff line change
Expand Up @@ -61,29 +61,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
conn = op.get_bind() # pylint: disable=no-member

statement = text(
"""
UPDATE db_dbnode SET type = 'calculation.job.simpleplugins.arithmetic.add.ArithmeticAddCalculation.'
WHERE type = 'calculation.job.arithmetic.add.ArithmeticAddCalculation.';
UPDATE db_dbnode SET type = 'calculation.job.simpleplugins.templatereplacer.TemplatereplacerCalculation.'
WHERE type = 'calculation.job.templatereplacer.TemplatereplacerCalculation.';
UPDATE db_dbnode SET process_type = 'aiida.calculations:simpleplugins.arithmetic.add'
WHERE process_type = 'aiida.calculations:arithmetic.add';
UPDATE db_dbnode SET process_type = 'aiida.calculations:simpleplugins.templatereplacer'
WHERE process_type = 'aiida.calculations:templatereplacer';
UPDATE db_dbnode SET attributes = jsonb_set(attributes, '{"input_plugin"}', '"simpleplugins.arithmetic.add"')
WHERE attributes @> '{"input_plugin": "arithmetic.add"}'
AND type = 'data.code.Code.';
UPDATE db_dbnode SET attributes = jsonb_set(attributes, '{"input_plugin"}', '"simpleplugins.templatereplacer"')
WHERE attributes @> '{"input_plugin": "templatereplacer"}'
AND type = 'data.code.Code.';
"""
)
conn.execute(statement)
raise NotImplementedError('Downgrade of 140c971ae0a3.')
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = '162b99bca4a2'
Expand All @@ -27,19 +25,10 @@


def upgrade():
"""Migrations for the upgrade."""
op.drop_table('db_dbcalcstate')


def downgrade():
op.create_table(
'db_dbcalcstate', sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('dbnode_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('state', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.Column('time', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
sa.ForeignKeyConstraint(['dbnode_id'], ['db_dbnode.id'],
name='db_dbcalcstate_dbnode_id_fkey',
ondelete='CASCADE',
initially='DEFERRED',
deferrable=True), sa.PrimaryKeyConstraint('id', name='db_dbcalcstate_pkey'),
sa.UniqueConstraint('dbnode_id', 'state', name='db_dbcalcstate_dbnode_id_state_key')
)
"""Migrations for the downgrade."""
raise NotImplementedError('Downgrade of 162b99bca4a2.')
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
# pylint: disable=invalid-name,no-member,import-error,no-name-in-module

from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1830c8430131'
Expand All @@ -30,10 +29,11 @@


def upgrade():
"""Migrations for the upgrade."""
op.drop_column('db_dbnode', 'nodeversion')
op.drop_column('db_dbnode', 'public')


def downgrade():
op.add_column('db_dbnode', sa.Column('public', sa.BOOLEAN(), autoincrement=False, nullable=True))
op.add_column('db_dbnode', sa.Column('nodeversion', sa.INTEGER(), autoincrement=False, nullable=True))
"""Migrations for the downgrade."""
raise NotImplementedError('Downgrade of 1830c8430131.')
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# For further information on the license, see the LICENSE.txt file #
# For further information please visit http://www.aiida.net #
###########################################################################
# pylint: disable=invalid-name
# pylint: disable=invalid-name,no-member,import-error,no-name-in-module
"""Remove legacy workflows
This is similar to migration django_0032
Expand All @@ -17,11 +17,7 @@
Create Date: 2019-04-03 17:11:44.073582
"""
# Remove when https://github.com/PyCQA/pylint/issues/1931 is fixed
# pylint: disable=no-member,import-error,no-name-in-module
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

from aiida.backends.sqlalchemy.migrations.utils.legacy_workflows import export_workflow_data

Expand Down Expand Up @@ -49,84 +45,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
op.create_table(
'db_dbworkflow',
sa.Column(
'id',
sa.INTEGER(),
server_default=sa.text("nextval('db_dbworkflow_id_seq'::regclass)"),
autoincrement=True,
nullable=False
),
sa.Column('uuid', postgresql.UUID(), autoincrement=False, nullable=True),
sa.Column('ctime', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
sa.Column('mtime', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('label', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.Column('description', sa.TEXT(), autoincrement=False, nullable=True),
sa.Column('nodeversion', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('lastsyncedversion', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('state', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.Column('report', sa.TEXT(), autoincrement=False, nullable=True),
sa.Column('module', sa.TEXT(), autoincrement=False, nullable=True),
sa.Column('module_class', sa.TEXT(), autoincrement=False, nullable=True),
sa.Column('script_path', sa.TEXT(), autoincrement=False, nullable=True),
sa.Column('script_md5', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.ForeignKeyConstraint(['user_id'], ['db_dbuser.id'], name='db_dbworkflow_user_id_fkey'),
sa.PrimaryKeyConstraint('id', name='db_dbworkflow_pkey'),
sa.UniqueConstraint('uuid', name='db_dbworkflow_uuid_key'),
postgresql_ignore_search_path=False
)
op.create_index('ix_db_dbworkflow_label', 'db_dbworkflow', ['label'], unique=False)
op.create_table(
'db_dbworkflowdata', sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column('parent_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('name', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.Column('time', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
sa.Column('data_type', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.Column('value_type', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.Column('json_value', sa.TEXT(), autoincrement=False, nullable=True),
sa.Column('aiida_obj_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.ForeignKeyConstraint(['aiida_obj_id'], ['db_dbnode.id'], name='db_dbworkflowdata_aiida_obj_id_fkey'),
sa.ForeignKeyConstraint(['parent_id'], ['db_dbworkflow.id'], name='db_dbworkflowdata_parent_id_fkey'),
sa.PrimaryKeyConstraint('id', name='db_dbworkflowdata_pkey'),
sa.UniqueConstraint('parent_id', 'name', 'data_type', name='db_dbworkflowdata_parent_id_name_data_type_key')
)
op.create_index('ix_db_dbworkflowdata_parent_id', 'db_dbworkflowdata', ['parent_id'], unique=False)
op.create_index('ix_db_dbworkflowdata_aiida_obj_id', 'db_dbworkflowdata', ['aiida_obj_id'], unique=False)
op.create_table(
'db_dbworkflowstep', sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column('parent_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('name', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.Column('time', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
sa.Column('nextcall', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.Column('state', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
sa.ForeignKeyConstraint(['parent_id'], ['db_dbworkflow.id'], name='db_dbworkflowstep_parent_id_fkey'),
sa.ForeignKeyConstraint(['user_id'], ['db_dbuser.id'], name='db_dbworkflowstep_user_id_fkey'),
sa.PrimaryKeyConstraint('id', name='db_dbworkflowstep_pkey'),
sa.UniqueConstraint('parent_id', 'name', name='db_dbworkflowstep_parent_id_name_key')
)
op.create_table(
'db_dbworkflowstep_calculations', sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column('dbworkflowstep_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('dbnode_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.ForeignKeyConstraint(['dbnode_id'], ['db_dbnode.id'], name='db_dbworkflowstep_calculations_dbnode_id_fkey'),
sa.ForeignKeyConstraint(['dbworkflowstep_id'], ['db_dbworkflowstep.id'],
name='db_dbworkflowstep_calculations_dbworkflowstep_id_fkey'),
sa.PrimaryKeyConstraint('id', name='db_dbworkflowstep_calculations_pkey'),
sa.UniqueConstraint('dbworkflowstep_id', 'dbnode_id', name='db_dbworkflowstep_calculations_id_dbnode_id_key')
)
op.create_table(
'db_dbworkflowstep_sub_workflows', sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
sa.Column('dbworkflowstep_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('dbworkflow_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.ForeignKeyConstraint(['dbworkflow_id'], ['db_dbworkflow.id'],
name='db_dbworkflowstep_sub_workflows_dbworkflow_id_fkey'),
sa.ForeignKeyConstraint(['dbworkflowstep_id'], ['db_dbworkflowstep.id'],
name='db_dbworkflowstep_sub_workflows_dbworkflowstep_id_fkey'),
sa.PrimaryKeyConstraint('id', name='db_dbworkflowstep_sub_workflows_pkey'),
sa.UniqueConstraint(
'dbworkflowstep_id', 'dbworkflow_id', name='db_dbworkflowstep_sub_workflows_id_dbworkflow__key'
)
)
raise NotImplementedError('Removal of legacy workflows is not reversible.')
Original file line number Diff line number Diff line change
Expand Up @@ -168,4 +168,4 @@ def upgrade(): # pylint: disable=too-many-statements

def downgrade():
"""Downgrade database schema."""
# No need to convert the values back to null
raise NotImplementedError('Downgrade of 1de112340b16.')
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
raise NotImplementedError('Downgrade of 1de112340b18.')
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
raise NotImplementedError('Migration of the file repository is not reversible.')
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@


def upgrade():
"""The upgrade migration actions."""
"""Migrations for the upgrade."""
from aiida.backends.sqlalchemy.migrations.utils import provenance_redesign

# Migrate calculation nodes by inferring the process type from the type string
Expand Down Expand Up @@ -109,32 +109,5 @@ def upgrade():


def downgrade():
"""The downgrade migration actions."""
op.execute(
"""
UPDATE db_dbnode SET type = 'calculation.job.JobCalculation.'
WHERE type = 'node.process.calculation.calcjob.CalcJobNode.';
UPDATE db_dbnode SET type = 'calculatison.inline.InlineCalculation.'
WHERE type = 'node.process.calculation.calcfunction.CalcFunctionNode.';
UPDATE db_dbnode SET type = 'calculation.function.FunctionCalculation.'
WHERE type = 'node.process.workflow.workfunction.WorkFunctionNode.';
UPDATE db_dbnode SET type = 'calculation.work.WorkCalculation.'
WHERE type = 'node.process.workflow.workchain.WorkChainNode.';
UPDATE db_dblink SET type = 'inputlink'
WHERE type = 'input_call' OR type = 'input_work';
UPDATE db_dblink SET type = 'calllink'
WHERE type = 'call_call' OR type = 'call_work';
UPDATE db_dblink SET type = 'createlink'
WHERE type = 'create';
UPDATE db_dblink SET type = 'returnlink'
WHERE type = 'return';
"""
)
"""Migrations for the downgrade."""
raise NotImplementedError('Downgrade of 239cea6d2452.')
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
raise NotImplementedError('Downgrade of 26d561acd560.')
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
raise NotImplementedError('Downgrade of 34a831f4286d.')
Original file line number Diff line number Diff line change
Expand Up @@ -54,26 +54,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
conn = op.get_bind()

# Set hidden=True in attributes if the extras contain hidden=True
statement = text(
"""
UPDATE db_dbnode SET attributes = jsonb_set(attributes, '{"hidden"}', to_jsonb(True))
WHERE type = 'code.Code.' AND extras @> '{"hidden": true}'
"""
)
conn.execute(statement)

# Set hidden=False in attributes if the extras contain hidden=False
statement = text(
"""
UPDATE db_dbnode SET attributes = jsonb_set(attributes, '{"hidden"}', to_jsonb(False))
WHERE type = 'code.Code.' AND extras @> '{"hidden": false}'
"""
)
conn.execute(statement)

# Delete the hidden key from the extras
statement = text("""UPDATE db_dbnode SET extras = extras-'hidden' WHERE type = 'code.Code.'""")
conn.execute(statement)
raise NotImplementedError('Downgrade of 35d4ee9a1b0e.')
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@


def upgrade():
"""Add unique key constraint to the UUID column."""
"""Migrations for the upgrade."""
op.create_unique_constraint('db_dblog_uuid_key', 'db_dblog', ['uuid'])


def downgrade():
"""Remove unique key constraint to the UUID column."""
"""Migrations for the downgrade."""
op.drop_constraint('db_dblog_uuid_key', 'db_dblog')
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
for table in tables:
op.drop_constraint(f'{table}_uuid_key', table)
raise NotImplementedError('Downgrade of 37f3d4882837.')
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,4 @@ def upgrade():

def downgrade():
"""Migrations for the downgrade."""
op.drop_constraint('db_dbcomputer_label_key', 'db_dbcomputer')
op.alter_column('db_dbcomputer', 'label', new_column_name='name') # pylint: disable=no-member
op.create_unique_constraint('db_dbcomputer_name_key', 'db_dbcomputer', ['name'])
raise NotImplementedError('Downgrade of 535039300e4a.')
Loading

0 comments on commit 842d47d

Please sign in to comment.