diff --git a/src/aiida/storage/migrations.py b/src/aiida/storage/migrations.py new file mode 100644 index 0000000000..c37cbab641 --- /dev/null +++ b/src/aiida/storage/migrations.py @@ -0,0 +1,8 @@ +"""Module with common resources related to storage migrations.""" + +TEMPLATE_INVALID_SCHEMA_VERSION = """ +Database schema version `{schema_version_database}` is incompatible with the required schema version `{schema_version_code}`. +To migrate the database schema version to the current one, run the following command: + + verdi -p {profile_name} storage migrate +""" # noqa: E501 diff --git a/src/aiida/storage/psql_dos/migrator.py b/src/aiida/storage/psql_dos/migrator.py index 3ea36b9307..5251fd49de 100644 --- a/src/aiida/storage/psql_dos/migrator.py +++ b/src/aiida/storage/psql_dos/migrator.py @@ -33,6 +33,7 @@ from aiida.common import exceptions from aiida.manage.configuration.profile import Profile from aiida.storage.log import MIGRATE_LOGGER +from aiida.storage.migrations import TEMPLATE_INVALID_SCHEMA_VERSION from aiida.storage.psql_dos.models.settings import DbSetting from aiida.storage.psql_dos.utils import create_sqlalchemy_engine @@ -46,13 +47,6 @@ verdi -p {profile_name} storage migrate """ -TEMPLATE_INVALID_SCHEMA_VERSION = """ -Database schema version `{schema_version_database}` is incompatible with the required schema version `{schema_version_code}`. -To migrate the database schema version to the current one, run the following command: - - verdi -p {profile_name} storage migrate -""" # noqa: E501 - ALEMBIC_REL_PATH = 'migrations' REPOSITORY_UUID_KEY = 'repository|uuid' diff --git a/src/aiida/storage/sqlite_dos/backend.py b/src/aiida/storage/sqlite_dos/backend.py index 3b13764b3d..7be70f4a1c 100644 --- a/src/aiida/storage/sqlite_dos/backend.py +++ b/src/aiida/storage/sqlite_dos/backend.py @@ -10,31 +10,36 @@ from __future__ import annotations +import pathlib from functools import cached_property, lru_cache from pathlib import Path from shutil import rmtree from typing import TYPE_CHECKING, Optional from uuid import uuid4 +from alembic.config import Config from disk_objectstore import Container, backup_utils from pydantic import BaseModel, Field, field_validator -from sqlalchemy import insert +from sqlalchemy import insert, inspect, select from sqlalchemy.orm import scoped_session, sessionmaker from aiida.common import exceptions from aiida.common.log import AIIDA_LOGGER -from aiida.manage import Profile +from aiida.manage.configuration.profile import Profile from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER from aiida.orm.implementation import BackendEntity +from aiida.storage.log import MIGRATE_LOGGER from aiida.storage.psql_dos.models.settings import DbSetting from aiida.storage.sqlite_zip import models, orm -from aiida.storage.sqlite_zip.migrator import get_schema_version_head from aiida.storage.sqlite_zip.utils import create_sqla_engine +from ..migrations import TEMPLATE_INVALID_SCHEMA_VERSION from ..psql_dos import PsqlDosBackend -from ..psql_dos.migrator import REPOSITORY_UUID_KEY, PsqlDosMigrator +from ..psql_dos.migrator import PsqlDosMigrator if TYPE_CHECKING: + from disk_objectstore import Container + from aiida.orm.entities import EntityTypes from aiida.repository.backend import DiskObjectStoreRepositoryBackend @@ -45,15 +50,26 @@ FILENAME_CONTAINER = 'container' +ALEMBIC_REL_PATH = 'migrations' + +REPOSITORY_UUID_KEY = 'repository|uuid' + + class SqliteDosMigrator(PsqlDosMigrator): - """Storage implementation using Sqlite database and disk-objectstore container. + """Class for validating and migrating `sqlite_dos` storage instances. - This storage backend is not recommended for use in production. The sqlite database is not the most performant and it - does not support all the ``QueryBuilder`` functionality that is supported by the ``core.psql_dos`` storage backend. - This storage is ideally suited for use cases that want to test or demo AiiDA as it requires no server but just a - folder on the local filesystem. + .. important:: This class should only be accessed via the storage backend class (apart from for test purposes) + + The class subclasses the ``PsqlDosMigrator``. It essentially changes two things in the implementation: + + * Changes the path to the migration version files. This allows custom migrations to be written for SQLite-based + storage plugins, which is necessary since the PSQL-based migrations may use syntax that is not compatible. + * The logic for validating the storage is significantly simplified since the SQLite-based storage plugins do not + have to take legacy Django-based implementations into account. """ + alembic_version_tbl_name = 'alembic_version' + def __init__(self, profile: Profile) -> None: filepath_database = Path(profile.storage_config['filepath']) / FILENAME_DATABASE filepath_database.touch() @@ -91,6 +107,86 @@ def initialise_database(self) -> None: context.stamp(context.script, 'main@head') # type: ignore[arg-type] self.connection.commit() + def get_schema_version_profile(self) -> Optional[str]: # type: ignore[override] + """Return the schema version of the backend instance for this profile. + + Note, the version will be None if the database is empty or is a legacy django database. + """ + with self._migration_context() as context: + return context.get_current_revision() + + @staticmethod + def _alembic_config(): + """Return an instance of an Alembic `Config`.""" + dirpath = pathlib.Path(__file__).resolve().parent + config = Config() + config.set_main_option('script_location', str(dirpath / ALEMBIC_REL_PATH)) + return config + + def validate_storage(self) -> None: + """Validate that the storage for this profile + + 1. That the database schema is at the head version, i.e. is compatible with the code API. + 2. That the repository ID is equal to the UUID set in the database + + :raises: :class:`aiida.common.exceptions.UnreachableStorage` if the storage cannot be connected to + :raises: :class:`aiida.common.exceptions.IncompatibleStorageSchema` + if the storage is not compatible with the code API. + :raises: :class:`aiida.common.exceptions.CorruptStorage` + if the repository ID is not equal to the UUID set in thedatabase. + """ + # check there is an alembic_version table from which to get the schema version + if not inspect(self.connection).has_table(self.alembic_version_tbl_name): + raise exceptions.IncompatibleStorageSchema('The database has no known version.') + + # now we can check that the alembic version is the latest + schema_version_code = self.get_schema_version_head() + schema_version_database = self.get_schema_version_profile() + if schema_version_database != schema_version_code: + raise exceptions.IncompatibleStorageSchema( + TEMPLATE_INVALID_SCHEMA_VERSION.format( + schema_version_database=schema_version_database, + schema_version_code=schema_version_code, + profile_name=self.profile.name, + ) + ) + + # finally, we check that the ID set within the disk-objectstore is equal to the one saved in the database, + # i.e. this container is indeed the one associated with the db + repository_uuid = self.get_repository_uuid() + stmt = select(DbSetting.val).where(DbSetting.key == REPOSITORY_UUID_KEY) + database_repository_uuid = self.connection.execute(stmt).scalar_one_or_none() + if database_repository_uuid is None: + raise exceptions.CorruptStorage('The database has no repository UUID set.') + if database_repository_uuid != repository_uuid: + raise exceptions.CorruptStorage( + f'The database has a repository UUID configured to {database_repository_uuid} ' + f"but the disk-objectstore's is {repository_uuid}." + ) + + @property + def is_database_initialised(self) -> bool: + """Return whether the database is initialised. + + This is the case if it contains the table that holds the schema version for alembic. + + :returns: ``True`` if the database is initialised, ``False`` otherwise. + """ + return inspect(self.connection).has_table(self.alembic_version_tbl_name) + + def migrate(self) -> None: + """Migrate the storage for this profile to the head version. + + :raises: :class:`~aiida.common.exceptions.UnreachableStorage` if the storage cannot be accessed. + :raises: :class:`~aiida.common.exceptions.StorageMigrationError` if the storage is not initialised. + """ + if not inspect(self.connection).has_table(self.alembic_version_tbl_name): + raise exceptions.StorageMigrationError('storage is uninitialised, cannot migrate.') + + MIGRATE_LOGGER.report('Migrating to the head of the main branch') + self.migrate_up('main@head') + self.connection.commit() + class SqliteDosStorage(PsqlDosBackend): """A lightweight storage that is easy to install. @@ -178,12 +274,9 @@ def get_repository(self) -> 'DiskObjectStoreRepositoryBackend': return DiskObjectStoreRepositoryBackend(container=self.get_container()) @classmethod - def version_head(cls) -> str: - return get_schema_version_head() - - @classmethod - def version_profile(cls, profile: Profile) -> str | None: - return get_schema_version_head() + def version_profile(cls, profile: Profile) -> Optional[str]: + with cls.migrator_context(profile) as migrator: + return migrator.get_schema_version_profile() def query(self) -> orm.SqliteQueryBuilder: return orm.SqliteQueryBuilder(self) diff --git a/src/aiida/storage/sqlite_dos/migrations/env.py b/src/aiida/storage/sqlite_dos/migrations/env.py new file mode 100644 index 0000000000..e2beb1ad9f --- /dev/null +++ b/src/aiida/storage/sqlite_dos/migrations/env.py @@ -0,0 +1,54 @@ +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Environment configuration to be used by alembic to perform database migrations.""" + +from alembic import context + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + The connection should have been passed to the config, which we use to configure the migration context. + """ + from aiida.storage.sqlite_zip.models import SqliteBase + + config = context.config + + connection = config.attributes.get('connection', None) + aiida_profile = config.attributes.get('aiida_profile', None) + on_version_apply = config.attributes.get('on_version_apply', None) + + if connection is None: + from aiida.common.exceptions import ConfigurationError + + raise ConfigurationError('An initialized connection is expected for the AiiDA online migrations.') + if aiida_profile is None: + from aiida.common.exceptions import ConfigurationError + + raise ConfigurationError('An aiida_profile is expected for the AiiDA online migrations.') + + context.configure( + connection=connection, + target_metadata=SqliteBase.metadata, + transaction_per_migration=True, + aiida_profile=aiida_profile, + on_version_apply=on_version_apply, + ) + + context.run_migrations() + + +try: + if context.is_offline_mode(): + raise NotImplementedError('This feature is not currently supported.') + + run_migrations_online() +except NameError: + # This will occur in an environment that is just compiling the documentation + pass diff --git a/src/aiida/storage/sqlite_dos/migrations/versions/main_0001_initial.py b/src/aiida/storage/sqlite_dos/migrations/versions/main_0001_initial.py new file mode 100644 index 0000000000..6af0887766 --- /dev/null +++ b/src/aiida/storage/sqlite_dos/migrations/versions/main_0001_initial.py @@ -0,0 +1,198 @@ +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Initial main branch schema + +This schema is mainly equivalent to the `main_0000_initial.py` schema of the `sqlite_zip` backend. Except that UUID +columns use ``String(32)`` instead of ``CHAR(32)``. + +Revision ID: main_0001 +Revises: +Create Date: 2024-05-29 +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.sqlite import JSON + +revision = 'main_0001' +down_revision = None +branch_labels = ('main',) +depends_on = None + + +def upgrade(): + """Migrations for the upgrade.""" + op.create_table( + 'db_dbcomputer', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('uuid', sa.String(32), nullable=False, unique=True), + sa.Column('label', sa.String(length=255), nullable=False, unique=True), + sa.Column('hostname', sa.String(length=255), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('scheduler_type', sa.String(length=255), nullable=False), + sa.Column('transport_type', sa.String(length=255), nullable=False), + sa.Column('metadata', JSON(), nullable=False), + ) + op.create_table( + 'db_dbuser', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('email', sa.String(length=254), nullable=False, unique=True), + sa.Column('first_name', sa.String(length=254), nullable=False), + sa.Column('last_name', sa.String(length=254), nullable=False), + sa.Column('institution', sa.String(length=254), nullable=False), + ) + op.create_table( + 'db_dbauthinfo', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('aiidauser_id', sa.Integer(), nullable=False, index=True), + sa.Column('dbcomputer_id', sa.Integer(), nullable=False, index=True), + sa.Column('metadata', JSON(), nullable=False), + sa.Column('auth_params', JSON(), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint( + ['aiidauser_id'], + ['db_dbuser.id'], + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True, + ), + sa.ForeignKeyConstraint( + ['dbcomputer_id'], + ['db_dbcomputer.id'], + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True, + ), + sa.UniqueConstraint('aiidauser_id', 'dbcomputer_id'), + ) + op.create_table( + 'db_dbgroup', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('uuid', sa.String(32), nullable=False, unique=True), + sa.Column('label', sa.String(length=255), nullable=False, index=True), + sa.Column('type_string', sa.String(length=255), nullable=False, index=True), + sa.Column('time', sa.DateTime(timezone=True), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('extras', JSON(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False, index=True), + sa.ForeignKeyConstraint( + ['user_id'], + ['db_dbuser.id'], + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True, + ), + sa.UniqueConstraint('label', 'type_string'), + ) + + op.create_table( + 'db_dbnode', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('uuid', sa.String(32), nullable=False, unique=True), + sa.Column('node_type', sa.String(length=255), nullable=False, index=True), + sa.Column('process_type', sa.String(length=255), nullable=True, index=True), + sa.Column('label', sa.String(length=255), nullable=False, index=True), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('ctime', sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column('mtime', sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column('attributes', JSON(), nullable=True), + sa.Column('extras', JSON(), nullable=True), + sa.Column('repository_metadata', JSON(), nullable=False), + sa.Column('dbcomputer_id', sa.Integer(), nullable=True, index=True), + sa.Column('user_id', sa.Integer(), nullable=False, index=True), + sa.ForeignKeyConstraint( + ['dbcomputer_id'], + ['db_dbcomputer.id'], + ondelete='RESTRICT', + initially='DEFERRED', + deferrable=True, + ), + sa.ForeignKeyConstraint( + ['user_id'], + ['db_dbuser.id'], + ondelete='restrict', + initially='DEFERRED', + deferrable=True, + ), + ) + + op.create_table( + 'db_dbcomment', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('uuid', sa.String(32), nullable=False, unique=True), + sa.Column('dbnode_id', sa.Integer(), nullable=False, index=True), + sa.Column('ctime', sa.DateTime(timezone=True), nullable=False), + sa.Column('mtime', sa.DateTime(timezone=True), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False, index=True), + sa.Column('content', sa.Text(), nullable=False), + sa.ForeignKeyConstraint( + ['dbnode_id'], + ['db_dbnode.id'], + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True, + ), + sa.ForeignKeyConstraint( + ['user_id'], + ['db_dbuser.id'], + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True, + ), + ) + + op.create_table( + 'db_dbgroup_dbnodes', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('dbnode_id', sa.Integer(), nullable=False, index=True), + sa.Column('dbgroup_id', sa.Integer(), nullable=False, index=True), + sa.ForeignKeyConstraint(['dbgroup_id'], ['db_dbgroup.id'], initially='DEFERRED', deferrable=True), + sa.ForeignKeyConstraint(['dbnode_id'], ['db_dbnode.id'], initially='DEFERRED', deferrable=True), + sa.UniqueConstraint('dbgroup_id', 'dbnode_id'), + ) + op.create_table( + 'db_dblink', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('input_id', sa.Integer(), nullable=False, index=True), + sa.Column('output_id', sa.Integer(), nullable=False, index=True), + sa.Column('label', sa.String(length=255), nullable=False, index=True), + sa.Column('type', sa.String(length=255), nullable=False, index=True), + sa.ForeignKeyConstraint(['input_id'], ['db_dbnode.id'], initially='DEFERRED', deferrable=True), + sa.ForeignKeyConstraint( + ['output_id'], + ['db_dbnode.id'], + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True, + ), + ) + + op.create_table( + 'db_dblog', + sa.Column('id', sa.Integer(), nullable=False, primary_key=True), + sa.Column('uuid', sa.String(32), nullable=False, unique=True), + sa.Column('time', sa.DateTime(timezone=True), nullable=False), + sa.Column('loggername', sa.String(length=255), nullable=False, index=True), + sa.Column('levelname', sa.String(length=50), nullable=False, index=True), + sa.Column('dbnode_id', sa.Integer(), nullable=False, index=True), + sa.Column('message', sa.Text(), nullable=False), + sa.Column('metadata', JSON(), nullable=False), + sa.ForeignKeyConstraint( + ['dbnode_id'], + ['db_dbnode.id'], + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True, + ), + ) + + +def downgrade(): + """Migrations for the downgrade.""" + raise NotImplementedError('Downgrade of main_0000.') diff --git a/src/aiida/storage/sqlite_dos/migrations/versions/main_0002_recompute_hash_calc_job_node.py b/src/aiida/storage/sqlite_dos/migrations/versions/main_0002_recompute_hash_calc_job_node.py new file mode 100644 index 0000000000..ae70c45c4c --- /dev/null +++ b/src/aiida/storage/sqlite_dos/migrations/versions/main_0002_recompute_hash_calc_job_node.py @@ -0,0 +1,84 @@ +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Drop the hashes for all ``CalcJobNode`` instances. + +The computed hash erroneously included the hash of the file repository. This was present as of v2.0 and so all nodes +created with versions since then will have incorrect hashes. + +Revision ID: main_0002 +Revises: main_0001 +Create Date: 2024-05-29 +""" + +from __future__ import annotations + +from aiida.common.log import AIIDA_LOGGER +from alembic import op + +LOGGER = AIIDA_LOGGER.getChild(__file__) + +revision = 'main_0002' +down_revision = 'main_0001' +branch_labels = None +depends_on = None + + +def drop_hashes(conn, hash_extra_key: str, entry_point_string: str | None = None) -> None: + """Drop hashes of nodes. + + Print warning only if the DB actually contains nodes. + + :param hash_extra_key: The key in the extras used to store the hash at the time of this migration. + :param entry_point_string: Optional entry point string of a node type to narrow the subset of nodes to reset. The + value should be a complete entry point string, e.g., ``aiida.node:process.calculation.calcjob`` to drop the hash + of all ``CalcJobNode`` rows. + """ + from aiida.orm.utils.node import get_type_string_from_class + from aiida.plugins import load_entry_point_from_string + from sqlalchemy.sql import text + + if entry_point_string is not None: + entry_point = load_entry_point_from_string(entry_point_string) + node_type = get_type_string_from_class(entry_point.__module__, entry_point.__name__) + else: + node_type = None + + if node_type: + statement_count = text(f"SELECT count(*) FROM db_dbnode WHERE node_type = '{node_type}';") + statement_update = text( + f"UPDATE db_dbnode SET extras = json_remove(db_dbnode.extras, '$.{hash_extra_key}') WHERE node_type = '{node_type}';" # noqa: E501 + ) + else: + statement_count = text('SELECT count(*) FROM db_dbnode;') + statement_update = text(f"UPDATE db_dbnode SET extras = json_remove(db_dbnode.extras, '$.{hash_extra_key}');") + + node_count = conn.execute(statement_count).fetchall()[0][0] + + if node_count > 0: + if entry_point_string: + msg = f'Invalidating the hashes of certain nodes. Please run `verdi node rehash -e {entry_point_string}`.' + else: + msg = 'Invalidating the hashes of all nodes. Please run `verdi node rehash`.' + LOGGER.warning(msg) + + conn.execute(statement_update) + + +def upgrade(): + """Migrations for the upgrade.""" + drop_hashes( + op.get_bind(), hash_extra_key='_aiida_hash', entry_point_string='aiida.node:process.calculation.calcjob' + ) + + +def downgrade(): + """Migrations for the downgrade.""" + drop_hashes( + op.get_bind(), hash_extra_key='_aiida_hash', entry_point_string='aiida.node:process.calculation.calcjob' + ) diff --git a/src/aiida/storage/sqlite_zip/migrations/env.py b/src/aiida/storage/sqlite_zip/migrations/env.py index 73abbd917b..5691a95568 100644 --- a/src/aiida/storage/sqlite_zip/migrations/env.py +++ b/src/aiida/storage/sqlite_zip/migrations/env.py @@ -6,7 +6,7 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### -"""Upper level SQLAlchemy migration funcitons.""" +"""Upper level SQLAlchemy migration functions.""" from alembic import context diff --git a/tests/cmdline/commands/test_status.py b/tests/cmdline/commands/test_status.py index d02aff07d2..a4b81dbfc6 100644 --- a/tests/cmdline/commands/test_status.py +++ b/tests/cmdline/commands/test_status.py @@ -68,6 +68,7 @@ def test_storage_unable_to_connect(run_cli_command): profile._attributes['storage']['config']['database_port'] = old_port +@pytest.mark.requires_psql def test_storage_incompatible(run_cli_command, monkeypatch): """Test `verdi status` when storage schema version is incompatible with that of the code.""" @@ -83,6 +84,7 @@ def storage_cls(*args, **kwargs): assert result.exit_code is ExitCode.CRITICAL +@pytest.mark.requires_psql def test_storage_corrupted(run_cli_command, monkeypatch): """Test `verdi status` when the storage is found to be corrupt (e.g. non-matching repository UUIDs).""" diff --git a/tests/storage/sqlite_dos/migrations/conftest.py b/tests/storage/sqlite_dos/migrations/conftest.py new file mode 100644 index 0000000000..bba974705f --- /dev/null +++ b/tests/storage/sqlite_dos/migrations/conftest.py @@ -0,0 +1,76 @@ +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Tests for the migration engine (Alembic) as well as for the AiiDA migrations for SQLAlchemy.""" + +import collections +import pathlib + +import pytest +from aiida.manage.configuration import Profile +from aiida.storage.sqlite_zip.utils import create_sqla_engine +from sqlalchemy import text + + +@pytest.fixture +def uninitialised_profile(tmp_path): + """Create a profile attached to an empty database and repository folder.""" + + yield Profile( + 'test_migrate', + { + 'test_profile': True, + 'storage': { + 'backend': 'core.sqlite_dos', + 'config': { + 'filepath': str(tmp_path), + }, + }, + 'process_control': {'backend': 'null', 'config': {}}, + }, + ) + + +def _generate_schema(profile: Profile) -> dict: + """Create a dict containing indexes of AiiDA tables.""" + with create_sqla_engine(pathlib.Path(profile.storage_config['filepath']) / 'database.sqlite').connect() as conn: + data = collections.defaultdict(list) + for type_, name, tbl_name, rootpage, sql in conn.execute(text('SELECT * FROM sqlite_master;')): + lines_sql = sql.strip().split('\n') if sql else [] + + # For an unknown reason, the ``sql`` is not deterministic as the order of the ``CONSTRAINTS`` rules seem to + # be in random order. To make sure they are always in the same order, they have to be ordered manually. + if type_ == 'table': + lines_constraints = [] + lines_other = [] + for line in lines_sql: + stripped = line.strip().strip(',') + + if 'CONSTRAINT' in stripped: + lines_constraints.append(stripped) + else: + lines_other.append(stripped) + + lines_sql = lines_other + sorted(lines_constraints) + data[type_].append((name, tbl_name, lines_sql)) + + for key in data.keys(): + data[key] = sorted(data[key], key=lambda v: v[0]) + + return dict(data) + + +@pytest.fixture +def reflect_schema(): + """A fixture to generate the schema of AiiDA tables for a given profile.""" + + def factory(profile: Profile) -> dict: + """Create a dict containing all tables and fields of AiiDA tables.""" + return _generate_schema(profile) + + return factory diff --git a/tests/storage/sqlite_dos/migrations/test_all_schema.py b/tests/storage/sqlite_dos/migrations/test_all_schema.py new file mode 100644 index 0000000000..51351f918e --- /dev/null +++ b/tests/storage/sqlite_dos/migrations/test_all_schema.py @@ -0,0 +1,49 @@ +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Basic tests for all migrations""" + +import pytest +from aiida.storage.sqlite_dos.backend import SqliteDosMigrator + + +@pytest.mark.parametrize('version', list(v for v in SqliteDosMigrator.get_schema_versions() if v.startswith('main'))) +def test_main(version, uninitialised_profile, reflect_schema, data_regression): + """Test that the migrations produce the expected database schema.""" + migrator = SqliteDosMigrator(uninitialised_profile) + migrator.migrate_up(f'main@{version}') + data_regression.check(reflect_schema(uninitialised_profile)) + + +def test_main_initialized(uninitialised_profile): + """Test that ``migrate`` properly stamps the new schema version when updating database with existing schema.""" + migrator = SqliteDosMigrator(uninitialised_profile) + + # Initialize database at first version of main branch + migrator.migrate_up('main@main_0001') + assert migrator.get_schema_version_profile() == 'main_0001' + migrator.close() + + # Reinitialize the migrator to make sure we are fetching actual state of database and not in-memory state and then + # migrate to head schema version. + migrator = SqliteDosMigrator(uninitialised_profile) + migrator.migrate() + migrator.close() + + # Reinitialize the migrator to make sure we are fetching actual state of database and not in-memory state and then + # assert that the database version is properly set to the head schema version + migrator = SqliteDosMigrator(uninitialised_profile) + assert migrator.get_schema_version_profile() == migrator.get_schema_version_head() + + +def test_head_vs_orm(uninitialised_profile, reflect_schema, data_regression): + """Test that the migrations produce the same database schema as the models.""" + migrator = SqliteDosMigrator(uninitialised_profile) + head_version = migrator.get_schema_version_head() + migrator.initialise() + data_regression.check(reflect_schema(uninitialised_profile), basename=f'test_head_vs_orm_{head_version}_') diff --git a/tests/storage/sqlite_dos/migrations/test_all_schema/test_head_vs_orm_main_0002_.yml b/tests/storage/sqlite_dos/migrations/test_all_schema/test_head_vs_orm_main_0002_.yml new file mode 100644 index 0000000000..b70a576550 --- /dev/null +++ b/tests/storage/sqlite_dos/migrations/test_all_schema/test_head_vs_orm_main_0002_.yml @@ -0,0 +1,269 @@ +index: +- - ix_db_dbauthinfo_db_dbauthinfo_aiidauser_id + - db_dbauthinfo + - - CREATE INDEX ix_db_dbauthinfo_db_dbauthinfo_aiidauser_id ON db_dbauthinfo (aiidauser_id) +- - ix_db_dbauthinfo_db_dbauthinfo_dbcomputer_id + - db_dbauthinfo + - - CREATE INDEX ix_db_dbauthinfo_db_dbauthinfo_dbcomputer_id ON db_dbauthinfo (dbcomputer_id) +- - ix_db_dbcomment_db_dbcomment_dbnode_id + - db_dbcomment + - - CREATE INDEX ix_db_dbcomment_db_dbcomment_dbnode_id ON db_dbcomment (dbnode_id) +- - ix_db_dbcomment_db_dbcomment_user_id + - db_dbcomment + - - CREATE INDEX ix_db_dbcomment_db_dbcomment_user_id ON db_dbcomment (user_id) +- - ix_db_dbgroup_db_dbgroup_label + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_label ON db_dbgroup (label) +- - ix_db_dbgroup_db_dbgroup_type_string + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_type_string ON db_dbgroup (type_string) +- - ix_db_dbgroup_db_dbgroup_user_id + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_user_id ON db_dbgroup (user_id) +- - ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbgroup_id + - db_dbgroup_dbnodes + - - CREATE INDEX ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbgroup_id ON db_dbgroup_dbnodes + (dbgroup_id) +- - ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbnode_id + - db_dbgroup_dbnodes + - - CREATE INDEX ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbnode_id ON db_dbgroup_dbnodes + (dbnode_id) +- - ix_db_dblink_db_dblink_input_id + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_input_id ON db_dblink (input_id) +- - ix_db_dblink_db_dblink_label + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_label ON db_dblink (label) +- - ix_db_dblink_db_dblink_output_id + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_output_id ON db_dblink (output_id) +- - ix_db_dblink_db_dblink_type + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_type ON db_dblink (type) +- - ix_db_dblog_db_dblog_dbnode_id + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_dbnode_id ON db_dblog (dbnode_id) +- - ix_db_dblog_db_dblog_levelname + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_levelname ON db_dblog (levelname) +- - ix_db_dblog_db_dblog_loggername + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_loggername ON db_dblog (loggername) +- - ix_db_dbnode_db_dbnode_ctime + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_ctime ON db_dbnode (ctime) +- - ix_db_dbnode_db_dbnode_dbcomputer_id + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_dbcomputer_id ON db_dbnode (dbcomputer_id) +- - ix_db_dbnode_db_dbnode_label + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_label ON db_dbnode (label) +- - ix_db_dbnode_db_dbnode_mtime + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_mtime ON db_dbnode (mtime) +- - ix_db_dbnode_db_dbnode_node_type + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_node_type ON db_dbnode (node_type) +- - ix_db_dbnode_db_dbnode_process_type + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_process_type ON db_dbnode (process_type) +- - ix_db_dbnode_db_dbnode_user_id + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_user_id ON db_dbnode (user_id) +- - sqlite_autoindex_alembic_version_1 + - alembic_version + - [] +- - sqlite_autoindex_db_dbauthinfo_1 + - db_dbauthinfo + - [] +- - sqlite_autoindex_db_dbcomment_1 + - db_dbcomment + - [] +- - sqlite_autoindex_db_dbcomputer_1 + - db_dbcomputer + - [] +- - sqlite_autoindex_db_dbcomputer_2 + - db_dbcomputer + - [] +- - sqlite_autoindex_db_dbgroup_1 + - db_dbgroup + - [] +- - sqlite_autoindex_db_dbgroup_2 + - db_dbgroup + - [] +- - sqlite_autoindex_db_dbgroup_dbnodes_1 + - db_dbgroup_dbnodes + - [] +- - sqlite_autoindex_db_dblog_1 + - db_dblog + - [] +- - sqlite_autoindex_db_dbnode_1 + - db_dbnode + - [] +- - sqlite_autoindex_db_dbsetting_1 + - db_dbsetting + - [] +- - sqlite_autoindex_db_dbuser_1 + - db_dbuser + - [] +table: +- - alembic_version + - alembic_version + - - CREATE TABLE alembic_version ( + - version_num VARCHAR(32) NOT NULL + - ) + - CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num) +- - db_dbauthinfo + - db_dbauthinfo + - - CREATE TABLE db_dbauthinfo ( + - id INTEGER NOT NULL + - aiidauser_id INTEGER NOT NULL + - dbcomputer_id INTEGER NOT NULL + - metadata JSON NOT NULL + - auth_params JSON NOT NULL + - enabled BOOLEAN NOT NULL + - ) + - CONSTRAINT db_dbauthinfo_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbauthinfo_aiidauser_id_db_dbuser FOREIGN KEY(aiidauser_id) + REFERENCES db_dbuser (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbauthinfo_dbcomputer_id_db_dbcomputer FOREIGN KEY(dbcomputer_id) + REFERENCES db_dbcomputer (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbauthinfo_aiidauser_id_dbcomputer_id UNIQUE (aiidauser_id, + dbcomputer_id) +- - db_dbcomment + - db_dbcomment + - - CREATE TABLE db_dbcomment ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - dbnode_id INTEGER NOT NULL + - ctime DATETIME NOT NULL + - mtime DATETIME NOT NULL + - user_id INTEGER NOT NULL + - content TEXT NOT NULL + - ) + - CONSTRAINT db_dbcomment_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbcomment_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbcomment_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES + db_dbuser (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbcomment_uuid UNIQUE (uuid) +- - db_dbcomputer + - db_dbcomputer + - - CREATE TABLE db_dbcomputer ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - label VARCHAR(255) NOT NULL + - hostname VARCHAR(255) NOT NULL + - description TEXT NOT NULL + - scheduler_type VARCHAR(255) NOT NULL + - transport_type VARCHAR(255) NOT NULL + - metadata JSON NOT NULL + - ) + - CONSTRAINT db_dbcomputer_pkey PRIMARY KEY (id) + - CONSTRAINT uq_db_dbcomputer_label UNIQUE (label) + - CONSTRAINT uq_db_dbcomputer_uuid UNIQUE (uuid) +- - db_dbgroup + - db_dbgroup + - - CREATE TABLE db_dbgroup ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - label VARCHAR(255) NOT NULL + - type_string VARCHAR(255) NOT NULL + - time DATETIME NOT NULL + - description TEXT NOT NULL + - extras JSON NOT NULL + - user_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbgroup_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbgroup_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES db_dbuser + (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbgroup_label_type_string UNIQUE (label, type_string) + - CONSTRAINT uq_db_dbgroup_uuid UNIQUE (uuid) +- - db_dbgroup_dbnodes + - db_dbgroup_dbnodes + - - CREATE TABLE db_dbgroup_dbnodes ( + - id INTEGER NOT NULL + - dbnode_id INTEGER NOT NULL + - dbgroup_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbgroup_dbnodes_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbgroup_dbnodes_dbgroup_id_db_dbgroup FOREIGN KEY(dbgroup_id) + REFERENCES db_dbgroup (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbgroup_dbnodes_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) + REFERENCES db_dbnode (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbgroup_dbnodes_dbgroup_id_dbnode_id UNIQUE (dbgroup_id, dbnode_id) +- - db_dblink + - db_dblink + - - CREATE TABLE db_dblink ( + - id INTEGER NOT NULL + - input_id INTEGER NOT NULL + - output_id INTEGER NOT NULL + - label VARCHAR(255) NOT NULL + - type VARCHAR(255) NOT NULL + - ) + - CONSTRAINT db_dblink_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dblink_input_id_db_dbnode FOREIGN KEY(input_id) REFERENCES + db_dbnode (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dblink_output_id_db_dbnode FOREIGN KEY(output_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED +- - db_dblog + - db_dblog + - - CREATE TABLE db_dblog ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - time DATETIME NOT NULL + - loggername VARCHAR(255) NOT NULL + - levelname VARCHAR(50) NOT NULL + - dbnode_id INTEGER NOT NULL + - message TEXT NOT NULL + - metadata JSON NOT NULL + - ) + - CONSTRAINT db_dblog_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dblog_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dblog_uuid UNIQUE (uuid) +- - db_dbnode + - db_dbnode + - - CREATE TABLE db_dbnode ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - node_type VARCHAR(255) NOT NULL + - process_type VARCHAR(255) + - label VARCHAR(255) NOT NULL + - description TEXT NOT NULL + - ctime DATETIME NOT NULL + - mtime DATETIME NOT NULL + - attributes JSON + - extras JSON + - repository_metadata JSON NOT NULL + - dbcomputer_id INTEGER + - user_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbnode_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbnode_dbcomputer_id_db_dbcomputer FOREIGN KEY(dbcomputer_id) + REFERENCES db_dbcomputer (id) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbnode_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES db_dbuser + (id) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbnode_uuid UNIQUE (uuid) +- - db_dbsetting + - db_dbsetting + - - CREATE TABLE db_dbsetting ( + - id INTEGER NOT NULL + - '"key" VARCHAR(1024) NOT NULL' + - val JSON + - description TEXT NOT NULL + - time DATETIME NOT NULL + - ) + - CONSTRAINT db_dbsetting_pkey PRIMARY KEY (id) + - CONSTRAINT uq_db_dbsetting_key UNIQUE ("key") +- - db_dbuser + - db_dbuser + - - CREATE TABLE db_dbuser ( + - id INTEGER NOT NULL + - email VARCHAR(254) NOT NULL + - first_name VARCHAR(254) NOT NULL + - last_name VARCHAR(254) NOT NULL + - institution VARCHAR(254) NOT NULL + - ) + - CONSTRAINT db_dbuser_pkey PRIMARY KEY (id) + - CONSTRAINT uq_db_dbuser_email UNIQUE (email) diff --git a/tests/storage/sqlite_dos/migrations/test_all_schema/test_main_main_0001_.yml b/tests/storage/sqlite_dos/migrations/test_all_schema/test_main_main_0001_.yml new file mode 100644 index 0000000000..3b49696512 --- /dev/null +++ b/tests/storage/sqlite_dos/migrations/test_all_schema/test_main_main_0001_.yml @@ -0,0 +1,255 @@ +index: +- - ix_db_dbauthinfo_db_dbauthinfo_aiidauser_id + - db_dbauthinfo + - - CREATE INDEX ix_db_dbauthinfo_db_dbauthinfo_aiidauser_id ON db_dbauthinfo (aiidauser_id) +- - ix_db_dbauthinfo_db_dbauthinfo_dbcomputer_id + - db_dbauthinfo + - - CREATE INDEX ix_db_dbauthinfo_db_dbauthinfo_dbcomputer_id ON db_dbauthinfo (dbcomputer_id) +- - ix_db_dbcomment_db_dbcomment_dbnode_id + - db_dbcomment + - - CREATE INDEX ix_db_dbcomment_db_dbcomment_dbnode_id ON db_dbcomment (dbnode_id) +- - ix_db_dbcomment_db_dbcomment_user_id + - db_dbcomment + - - CREATE INDEX ix_db_dbcomment_db_dbcomment_user_id ON db_dbcomment (user_id) +- - ix_db_dbgroup_db_dbgroup_label + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_label ON db_dbgroup (label) +- - ix_db_dbgroup_db_dbgroup_type_string + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_type_string ON db_dbgroup (type_string) +- - ix_db_dbgroup_db_dbgroup_user_id + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_user_id ON db_dbgroup (user_id) +- - ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbgroup_id + - db_dbgroup_dbnodes + - - CREATE INDEX ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbgroup_id ON db_dbgroup_dbnodes + (dbgroup_id) +- - ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbnode_id + - db_dbgroup_dbnodes + - - CREATE INDEX ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbnode_id ON db_dbgroup_dbnodes + (dbnode_id) +- - ix_db_dblink_db_dblink_input_id + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_input_id ON db_dblink (input_id) +- - ix_db_dblink_db_dblink_label + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_label ON db_dblink (label) +- - ix_db_dblink_db_dblink_output_id + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_output_id ON db_dblink (output_id) +- - ix_db_dblink_db_dblink_type + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_type ON db_dblink (type) +- - ix_db_dblog_db_dblog_dbnode_id + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_dbnode_id ON db_dblog (dbnode_id) +- - ix_db_dblog_db_dblog_levelname + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_levelname ON db_dblog (levelname) +- - ix_db_dblog_db_dblog_loggername + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_loggername ON db_dblog (loggername) +- - ix_db_dbnode_db_dbnode_ctime + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_ctime ON db_dbnode (ctime) +- - ix_db_dbnode_db_dbnode_dbcomputer_id + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_dbcomputer_id ON db_dbnode (dbcomputer_id) +- - ix_db_dbnode_db_dbnode_label + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_label ON db_dbnode (label) +- - ix_db_dbnode_db_dbnode_mtime + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_mtime ON db_dbnode (mtime) +- - ix_db_dbnode_db_dbnode_node_type + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_node_type ON db_dbnode (node_type) +- - ix_db_dbnode_db_dbnode_process_type + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_process_type ON db_dbnode (process_type) +- - ix_db_dbnode_db_dbnode_user_id + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_user_id ON db_dbnode (user_id) +- - sqlite_autoindex_alembic_version_1 + - alembic_version + - [] +- - sqlite_autoindex_db_dbauthinfo_1 + - db_dbauthinfo + - [] +- - sqlite_autoindex_db_dbcomment_1 + - db_dbcomment + - [] +- - sqlite_autoindex_db_dbcomputer_1 + - db_dbcomputer + - [] +- - sqlite_autoindex_db_dbcomputer_2 + - db_dbcomputer + - [] +- - sqlite_autoindex_db_dbgroup_1 + - db_dbgroup + - [] +- - sqlite_autoindex_db_dbgroup_2 + - db_dbgroup + - [] +- - sqlite_autoindex_db_dbgroup_dbnodes_1 + - db_dbgroup_dbnodes + - [] +- - sqlite_autoindex_db_dblog_1 + - db_dblog + - [] +- - sqlite_autoindex_db_dbnode_1 + - db_dbnode + - [] +- - sqlite_autoindex_db_dbuser_1 + - db_dbuser + - [] +table: +- - alembic_version + - alembic_version + - - CREATE TABLE alembic_version ( + - version_num VARCHAR(32) NOT NULL + - ) + - CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num) +- - db_dbauthinfo + - db_dbauthinfo + - - CREATE TABLE db_dbauthinfo ( + - id INTEGER NOT NULL + - aiidauser_id INTEGER NOT NULL + - dbcomputer_id INTEGER NOT NULL + - metadata JSON NOT NULL + - auth_params JSON NOT NULL + - enabled BOOLEAN NOT NULL + - ) + - CONSTRAINT db_dbauthinfo_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbauthinfo_aiidauser_id_db_dbuser FOREIGN KEY(aiidauser_id) + REFERENCES db_dbuser (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbauthinfo_dbcomputer_id_db_dbcomputer FOREIGN KEY(dbcomputer_id) + REFERENCES db_dbcomputer (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbauthinfo_aiidauser_id_dbcomputer_id UNIQUE (aiidauser_id, + dbcomputer_id) +- - db_dbcomment + - db_dbcomment + - - CREATE TABLE db_dbcomment ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - dbnode_id INTEGER NOT NULL + - ctime DATETIME NOT NULL + - mtime DATETIME NOT NULL + - user_id INTEGER NOT NULL + - content TEXT NOT NULL + - ) + - CONSTRAINT db_dbcomment_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbcomment_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbcomment_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES + db_dbuser (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbcomment_uuid UNIQUE (uuid) +- - db_dbcomputer + - db_dbcomputer + - - CREATE TABLE db_dbcomputer ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - label VARCHAR(255) NOT NULL + - hostname VARCHAR(255) NOT NULL + - description TEXT NOT NULL + - scheduler_type VARCHAR(255) NOT NULL + - transport_type VARCHAR(255) NOT NULL + - metadata JSON NOT NULL + - ) + - CONSTRAINT db_dbcomputer_pkey PRIMARY KEY (id) + - CONSTRAINT uq_db_dbcomputer_label UNIQUE (label) + - CONSTRAINT uq_db_dbcomputer_uuid UNIQUE (uuid) +- - db_dbgroup + - db_dbgroup + - - CREATE TABLE db_dbgroup ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - label VARCHAR(255) NOT NULL + - type_string VARCHAR(255) NOT NULL + - time DATETIME NOT NULL + - description TEXT NOT NULL + - extras JSON NOT NULL + - user_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbgroup_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbgroup_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES db_dbuser + (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbgroup_label_type_string UNIQUE (label, type_string) + - CONSTRAINT uq_db_dbgroup_uuid UNIQUE (uuid) +- - db_dbgroup_dbnodes + - db_dbgroup_dbnodes + - - CREATE TABLE db_dbgroup_dbnodes ( + - id INTEGER NOT NULL + - dbnode_id INTEGER NOT NULL + - dbgroup_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbgroup_dbnodes_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbgroup_dbnodes_dbgroup_id_db_dbgroup FOREIGN KEY(dbgroup_id) + REFERENCES db_dbgroup (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbgroup_dbnodes_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) + REFERENCES db_dbnode (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbgroup_dbnodes_dbgroup_id_dbnode_id UNIQUE (dbgroup_id, dbnode_id) +- - db_dblink + - db_dblink + - - CREATE TABLE db_dblink ( + - id INTEGER NOT NULL + - input_id INTEGER NOT NULL + - output_id INTEGER NOT NULL + - label VARCHAR(255) NOT NULL + - type VARCHAR(255) NOT NULL + - ) + - CONSTRAINT db_dblink_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dblink_input_id_db_dbnode FOREIGN KEY(input_id) REFERENCES + db_dbnode (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dblink_output_id_db_dbnode FOREIGN KEY(output_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED +- - db_dblog + - db_dblog + - - CREATE TABLE db_dblog ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - time DATETIME NOT NULL + - loggername VARCHAR(255) NOT NULL + - levelname VARCHAR(50) NOT NULL + - dbnode_id INTEGER NOT NULL + - message TEXT NOT NULL + - metadata JSON NOT NULL + - ) + - CONSTRAINT db_dblog_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dblog_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dblog_uuid UNIQUE (uuid) +- - db_dbnode + - db_dbnode + - - CREATE TABLE db_dbnode ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - node_type VARCHAR(255) NOT NULL + - process_type VARCHAR(255) + - label VARCHAR(255) NOT NULL + - description TEXT NOT NULL + - ctime DATETIME NOT NULL + - mtime DATETIME NOT NULL + - attributes JSON + - extras JSON + - repository_metadata JSON NOT NULL + - dbcomputer_id INTEGER + - user_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbnode_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbnode_dbcomputer_id_db_dbcomputer FOREIGN KEY(dbcomputer_id) + REFERENCES db_dbcomputer (id) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbnode_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES db_dbuser + (id) ON DELETE restrict DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbnode_uuid UNIQUE (uuid) +- - db_dbuser + - db_dbuser + - - CREATE TABLE db_dbuser ( + - id INTEGER NOT NULL + - email VARCHAR(254) NOT NULL + - first_name VARCHAR(254) NOT NULL + - last_name VARCHAR(254) NOT NULL + - institution VARCHAR(254) NOT NULL + - ) + - CONSTRAINT db_dbuser_pkey PRIMARY KEY (id) + - CONSTRAINT uq_db_dbuser_email UNIQUE (email) diff --git a/tests/storage/sqlite_dos/migrations/test_all_schema/test_main_main_0002_.yml b/tests/storage/sqlite_dos/migrations/test_all_schema/test_main_main_0002_.yml new file mode 100644 index 0000000000..3b49696512 --- /dev/null +++ b/tests/storage/sqlite_dos/migrations/test_all_schema/test_main_main_0002_.yml @@ -0,0 +1,255 @@ +index: +- - ix_db_dbauthinfo_db_dbauthinfo_aiidauser_id + - db_dbauthinfo + - - CREATE INDEX ix_db_dbauthinfo_db_dbauthinfo_aiidauser_id ON db_dbauthinfo (aiidauser_id) +- - ix_db_dbauthinfo_db_dbauthinfo_dbcomputer_id + - db_dbauthinfo + - - CREATE INDEX ix_db_dbauthinfo_db_dbauthinfo_dbcomputer_id ON db_dbauthinfo (dbcomputer_id) +- - ix_db_dbcomment_db_dbcomment_dbnode_id + - db_dbcomment + - - CREATE INDEX ix_db_dbcomment_db_dbcomment_dbnode_id ON db_dbcomment (dbnode_id) +- - ix_db_dbcomment_db_dbcomment_user_id + - db_dbcomment + - - CREATE INDEX ix_db_dbcomment_db_dbcomment_user_id ON db_dbcomment (user_id) +- - ix_db_dbgroup_db_dbgroup_label + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_label ON db_dbgroup (label) +- - ix_db_dbgroup_db_dbgroup_type_string + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_type_string ON db_dbgroup (type_string) +- - ix_db_dbgroup_db_dbgroup_user_id + - db_dbgroup + - - CREATE INDEX ix_db_dbgroup_db_dbgroup_user_id ON db_dbgroup (user_id) +- - ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbgroup_id + - db_dbgroup_dbnodes + - - CREATE INDEX ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbgroup_id ON db_dbgroup_dbnodes + (dbgroup_id) +- - ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbnode_id + - db_dbgroup_dbnodes + - - CREATE INDEX ix_db_dbgroup_dbnodes_db_dbgroup_dbnodes_dbnode_id ON db_dbgroup_dbnodes + (dbnode_id) +- - ix_db_dblink_db_dblink_input_id + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_input_id ON db_dblink (input_id) +- - ix_db_dblink_db_dblink_label + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_label ON db_dblink (label) +- - ix_db_dblink_db_dblink_output_id + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_output_id ON db_dblink (output_id) +- - ix_db_dblink_db_dblink_type + - db_dblink + - - CREATE INDEX ix_db_dblink_db_dblink_type ON db_dblink (type) +- - ix_db_dblog_db_dblog_dbnode_id + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_dbnode_id ON db_dblog (dbnode_id) +- - ix_db_dblog_db_dblog_levelname + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_levelname ON db_dblog (levelname) +- - ix_db_dblog_db_dblog_loggername + - db_dblog + - - CREATE INDEX ix_db_dblog_db_dblog_loggername ON db_dblog (loggername) +- - ix_db_dbnode_db_dbnode_ctime + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_ctime ON db_dbnode (ctime) +- - ix_db_dbnode_db_dbnode_dbcomputer_id + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_dbcomputer_id ON db_dbnode (dbcomputer_id) +- - ix_db_dbnode_db_dbnode_label + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_label ON db_dbnode (label) +- - ix_db_dbnode_db_dbnode_mtime + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_mtime ON db_dbnode (mtime) +- - ix_db_dbnode_db_dbnode_node_type + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_node_type ON db_dbnode (node_type) +- - ix_db_dbnode_db_dbnode_process_type + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_process_type ON db_dbnode (process_type) +- - ix_db_dbnode_db_dbnode_user_id + - db_dbnode + - - CREATE INDEX ix_db_dbnode_db_dbnode_user_id ON db_dbnode (user_id) +- - sqlite_autoindex_alembic_version_1 + - alembic_version + - [] +- - sqlite_autoindex_db_dbauthinfo_1 + - db_dbauthinfo + - [] +- - sqlite_autoindex_db_dbcomment_1 + - db_dbcomment + - [] +- - sqlite_autoindex_db_dbcomputer_1 + - db_dbcomputer + - [] +- - sqlite_autoindex_db_dbcomputer_2 + - db_dbcomputer + - [] +- - sqlite_autoindex_db_dbgroup_1 + - db_dbgroup + - [] +- - sqlite_autoindex_db_dbgroup_2 + - db_dbgroup + - [] +- - sqlite_autoindex_db_dbgroup_dbnodes_1 + - db_dbgroup_dbnodes + - [] +- - sqlite_autoindex_db_dblog_1 + - db_dblog + - [] +- - sqlite_autoindex_db_dbnode_1 + - db_dbnode + - [] +- - sqlite_autoindex_db_dbuser_1 + - db_dbuser + - [] +table: +- - alembic_version + - alembic_version + - - CREATE TABLE alembic_version ( + - version_num VARCHAR(32) NOT NULL + - ) + - CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num) +- - db_dbauthinfo + - db_dbauthinfo + - - CREATE TABLE db_dbauthinfo ( + - id INTEGER NOT NULL + - aiidauser_id INTEGER NOT NULL + - dbcomputer_id INTEGER NOT NULL + - metadata JSON NOT NULL + - auth_params JSON NOT NULL + - enabled BOOLEAN NOT NULL + - ) + - CONSTRAINT db_dbauthinfo_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbauthinfo_aiidauser_id_db_dbuser FOREIGN KEY(aiidauser_id) + REFERENCES db_dbuser (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbauthinfo_dbcomputer_id_db_dbcomputer FOREIGN KEY(dbcomputer_id) + REFERENCES db_dbcomputer (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbauthinfo_aiidauser_id_dbcomputer_id UNIQUE (aiidauser_id, + dbcomputer_id) +- - db_dbcomment + - db_dbcomment + - - CREATE TABLE db_dbcomment ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - dbnode_id INTEGER NOT NULL + - ctime DATETIME NOT NULL + - mtime DATETIME NOT NULL + - user_id INTEGER NOT NULL + - content TEXT NOT NULL + - ) + - CONSTRAINT db_dbcomment_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbcomment_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbcomment_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES + db_dbuser (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbcomment_uuid UNIQUE (uuid) +- - db_dbcomputer + - db_dbcomputer + - - CREATE TABLE db_dbcomputer ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - label VARCHAR(255) NOT NULL + - hostname VARCHAR(255) NOT NULL + - description TEXT NOT NULL + - scheduler_type VARCHAR(255) NOT NULL + - transport_type VARCHAR(255) NOT NULL + - metadata JSON NOT NULL + - ) + - CONSTRAINT db_dbcomputer_pkey PRIMARY KEY (id) + - CONSTRAINT uq_db_dbcomputer_label UNIQUE (label) + - CONSTRAINT uq_db_dbcomputer_uuid UNIQUE (uuid) +- - db_dbgroup + - db_dbgroup + - - CREATE TABLE db_dbgroup ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - label VARCHAR(255) NOT NULL + - type_string VARCHAR(255) NOT NULL + - time DATETIME NOT NULL + - description TEXT NOT NULL + - extras JSON NOT NULL + - user_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbgroup_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbgroup_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES db_dbuser + (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbgroup_label_type_string UNIQUE (label, type_string) + - CONSTRAINT uq_db_dbgroup_uuid UNIQUE (uuid) +- - db_dbgroup_dbnodes + - db_dbgroup_dbnodes + - - CREATE TABLE db_dbgroup_dbnodes ( + - id INTEGER NOT NULL + - dbnode_id INTEGER NOT NULL + - dbgroup_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbgroup_dbnodes_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbgroup_dbnodes_dbgroup_id_db_dbgroup FOREIGN KEY(dbgroup_id) + REFERENCES db_dbgroup (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbgroup_dbnodes_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) + REFERENCES db_dbnode (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbgroup_dbnodes_dbgroup_id_dbnode_id UNIQUE (dbgroup_id, dbnode_id) +- - db_dblink + - db_dblink + - - CREATE TABLE db_dblink ( + - id INTEGER NOT NULL + - input_id INTEGER NOT NULL + - output_id INTEGER NOT NULL + - label VARCHAR(255) NOT NULL + - type VARCHAR(255) NOT NULL + - ) + - CONSTRAINT db_dblink_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dblink_input_id_db_dbnode FOREIGN KEY(input_id) REFERENCES + db_dbnode (id) DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dblink_output_id_db_dbnode FOREIGN KEY(output_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED +- - db_dblog + - db_dblog + - - CREATE TABLE db_dblog ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - time DATETIME NOT NULL + - loggername VARCHAR(255) NOT NULL + - levelname VARCHAR(50) NOT NULL + - dbnode_id INTEGER NOT NULL + - message TEXT NOT NULL + - metadata JSON NOT NULL + - ) + - CONSTRAINT db_dblog_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dblog_dbnode_id_db_dbnode FOREIGN KEY(dbnode_id) REFERENCES + db_dbnode (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dblog_uuid UNIQUE (uuid) +- - db_dbnode + - db_dbnode + - - CREATE TABLE db_dbnode ( + - id INTEGER NOT NULL + - uuid VARCHAR(32) NOT NULL + - node_type VARCHAR(255) NOT NULL + - process_type VARCHAR(255) + - label VARCHAR(255) NOT NULL + - description TEXT NOT NULL + - ctime DATETIME NOT NULL + - mtime DATETIME NOT NULL + - attributes JSON + - extras JSON + - repository_metadata JSON NOT NULL + - dbcomputer_id INTEGER + - user_id INTEGER NOT NULL + - ) + - CONSTRAINT db_dbnode_pkey PRIMARY KEY (id) + - CONSTRAINT fk_db_dbnode_dbcomputer_id_db_dbcomputer FOREIGN KEY(dbcomputer_id) + REFERENCES db_dbcomputer (id) ON DELETE RESTRICT DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT fk_db_dbnode_user_id_db_dbuser FOREIGN KEY(user_id) REFERENCES db_dbuser + (id) ON DELETE restrict DEFERRABLE INITIALLY DEFERRED + - CONSTRAINT uq_db_dbnode_uuid UNIQUE (uuid) +- - db_dbuser + - db_dbuser + - - CREATE TABLE db_dbuser ( + - id INTEGER NOT NULL + - email VARCHAR(254) NOT NULL + - first_name VARCHAR(254) NOT NULL + - last_name VARCHAR(254) NOT NULL + - institution VARCHAR(254) NOT NULL + - ) + - CONSTRAINT db_dbuser_pkey PRIMARY KEY (id) + - CONSTRAINT uq_db_dbuser_email UNIQUE (email)