From 42c8f2b2ec9c86301c704cf765d9c520eae0b8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= <9308656+matrss@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:25:40 +0200 Subject: [PATCH] Use flask-migrate for database migrations (#2432) With this flask-migrate is used on application setup to automatically migrate the configured database to the latest revision shipped with MSColab. This will make manual migrations in production environments unnecessary in the future. Also included is an upgrade path from databases which were previously manually managed. This entails generating a new database and copying over all data. --- docs/development.rst | 20 + docs/mscolab.rst | 87 +---- mslib/mscolab/app/__init__.py | 20 +- mslib/mscolab/conf.py | 3 + mslib/mscolab/custom_migration_types.py | 27 ++ mslib/mscolab/migrations/env.py | 4 +- mslib/mscolab/migrations/script.py.mako | 1 + .../migrations/versions/83993fcdf5ef_.py | 92 ----- .../922e4d9c94e2_to_version_10_0_0.py | 33 ++ ...92e_to_version_8_3_5_initial_migration.py} | 57 +-- .../versions/c171019fe3ee_to_version_9_0_0.py | 35 ++ mslib/mscolab/mscolab.py | 46 +-- mslib/mscolab/seed.py | 341 +++++++++--------- mslib/mscolab/server.py | 65 ++++ setup.cfg | 2 +- tests/_test_mscolab/test_migrations.py | 133 +++++++ tests/_test_mscolab/test_mscolab.py | 2 +- tests/fixtures.py | 6 +- 18 files changed, 585 insertions(+), 389 deletions(-) create mode 100644 mslib/mscolab/custom_migration_types.py delete mode 100644 mslib/mscolab/migrations/versions/83993fcdf5ef_.py create mode 100644 mslib/mscolab/migrations/versions/922e4d9c94e2_to_version_10_0_0.py rename mslib/mscolab/migrations/versions/{cd8b7108713a_.py => 92eaba86a92e_to_version_8_3_5_initial_migration.py} (52%) create mode 100644 mslib/mscolab/migrations/versions/c171019fe3ee_to_version_9_0_0.py create mode 100644 tests/_test_mscolab/test_migrations.py diff --git a/docs/development.rst b/docs/development.rst index c7968a2d0..ecbf42d07 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -399,6 +399,26 @@ like e.g. a running MSColab server or a QApplication instance for GUI tests, are collected in :mod:`tests.fixtures` in the form of pytest fixtures that can be requested as needed in tests. +Changing the database model +--------------------------- + +Changing the database model requires adding a corresponding migration script to MSS, +so that existing databases can be migrated automatically. + +To generate such a migration script you can run:: + + flask --app mslib.mscolab.app db migrate -d mslib/mscolab/migrations -m "To version " + +Depending on the complexity of the changes that were made, +the generated migration script might need some tweaking. + +If there is already a migration script for the next release, +then please incorporate the generated migration script into this existing one, +instead of adding a new one. +You can still generate a script with the above command first +to get a starting point for the changes. + + Pushing your changes -------------------- diff --git a/docs/mscolab.rst b/docs/mscolab.rst index ec482b220..80459d6ad 100644 --- a/docs/mscolab.rst +++ b/docs/mscolab.rst @@ -150,75 +150,24 @@ by `pg_dump `_ using a pg_dump -d mscolab -f "/home/mscolab/dump/$timestamp.sql" -Data Base Migration from version 8 -.................................. - -.. important:: - This manual migration on the server side by a user is deprecated and will become removed with version 10.0.0. - With version 10.0.0, the initialization of the database will be refactored and migrations will be performed automatically when mscolab is started - -For an easy way to update the database scheme we implemented `flask migrate `_. - -You have to create based on your configuration a migration script and call that afterwards. :: - - mamba activate instance - cd ~/INSTANCE/config - export PYTHONPATH=`pwd` - cd ~/INSTANCE/wsgi - flask --app mscolab.py db init - flask --app mscolab.py db migrate -m "To version 9.0.0" - flask --app mscolab.py db upgrade - -The migration script builder does the base but the created script needs first allow nullable so that we afterwards set the default for the existing data. -Use this as an example for your script :: - - """To version 9.0.0 - - Revision ID: e62d08ce88a4 - Revises: 27a026b0daec - Create Date: 2024-06-07 16:53:43.314338 - - """ - from alembic import op - import sqlalchemy as sa - - - # revision identifiers, used by Alembic. - revision = 'e62d08ce88a4' - down_revision = '27a026b0daec' - branch_labels = None - depends_on = None - - - def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.add_column(sa.Column('authentication_backend', sa.String(length=255), nullable=True)) - op.execute('UPDATE users SET authentication_backend = \'local\' WHERE authentication_backend IS NULL') - - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.alter_column('authentication_backend', existing_type=sa.String(length=255), nullable=False) - batch_op.drop_constraint('users_password_key', type_='unique') - - # ### end Alembic commands ### - - - def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.create_unique_constraint('users_password_key', ['password']) - batch_op.drop_column('authentication_backend') - - # ### end Alembic commands ### - - -The output looks like :: - - ~/INSTANCE/wsgi$ flask --app mscolab.py db upgrade - INFO [alembic.runtime.migration] Context impl SQLiteImpl. - INFO [alembic.runtime.migration] Will assume non-transactional DDL. - INFO [alembic.runtime.migration] Running upgrade -> e62d08ce88a, To version 9.0.0 - +Database Migration from Version 8 or 9 +................................. + +From v10 onwards MSColab uses `Flask-Migrate ` to automatically deal with database migrations. +To upgrade from v8 or v9 a recreation of the database and subsequent copy of existing data is necessary. +To do this follow these steps: + +#. Stop MSColab completely, no process interacting with the MSColab database should remain running +#. **Make a backup of your existing database** +#. Set ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM`` to your existing database +#. Set ``SQLALCHEMY_DB_URI`` to a new database +#. If you are not using SQLite: create the new database +#. Start MSColab +#. Check that everything was migrated successfully +#. Unset ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM`` + +If you want to keep using your old database URI you can first rename your existing database so that it has a different URI +and just set ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM`` to that. Steps to use the MSColab UI features diff --git a/mslib/mscolab/app/__init__.py b/mslib/mscolab/app/__init__.py index 9cf33a150..374383e1c 100644 --- a/mslib/mscolab/app/__init__.py +++ b/mslib/mscolab/app/__init__.py @@ -25,6 +25,7 @@ """ import os +import sqlalchemy from flask_migrate import Migrate @@ -62,8 +63,23 @@ APP.config['MAIL_USE_TLS'] = getattr(mscolab_settings, "MAIL_USE_TLS", None) APP.config['MAIL_USE_SSL'] = getattr(mscolab_settings, "MAIL_USE_SSL", None) -db = SQLAlchemy(APP) -migrate = Migrate(APP, db, render_as_batch=True) +db = SQLAlchemy( + metadata=sqlalchemy.MetaData( + naming_convention={ + # For reference: https://alembic.sqlalchemy.org/en/latest/naming.html#the-importance-of-naming-constraints + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_`%(constraint_name)s`", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", + }, + ), +) +db.init_app(APP) +import mslib.mscolab.models + +migrate = Migrate(render_as_batch=True, user_module_prefix="cu.") +migrate.init_app(APP, db) def get_topmenu(): diff --git a/mslib/mscolab/conf.py b/mslib/mscolab/conf.py index af083085c..4321f7ab2 100644 --- a/mslib/mscolab/conf.py +++ b/mslib/mscolab/conf.py @@ -63,6 +63,9 @@ class default_mscolab_settings: # MYSQL CONNECTION STRING: "mysql+pymysql://:@:/?charset=utf8mb4" SQLALCHEMY_DB_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'mscolab.db') + # SQLAlchemy connection string to migrate data from, if set + SQLALCHEMY_DB_URI_TO_MIGRATE_FROM = None + # Set to True for testing and False for production SQLALCHEMY_ECHO = False diff --git a/mslib/mscolab/custom_migration_types.py b/mslib/mscolab/custom_migration_types.py new file mode 100644 index 000000000..f54f3a90d --- /dev/null +++ b/mslib/mscolab/custom_migration_types.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" + + mslib.mscolab.custom_migration_types + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Consolidated module of all custom/non-standard database types used in MSColab. + + This file is part of MSS. + + :copyright: Copyright 2024 Matthias Riße + :copyright: Copyright 2024 by the MSS team, see AUTHORS. + :license: APACHE-2.0, see LICENSE for details. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +from mslib.mscolab.models import AwareDateTime # noqa: F401 diff --git a/mslib/mscolab/migrations/env.py b/mslib/mscolab/migrations/env.py index 9bb35bb93..7b3a3322d 100644 --- a/mslib/mscolab/migrations/env.py +++ b/mslib/mscolab/migrations/env.py @@ -45,7 +45,7 @@ # target_metadata = mymodel.Base.metadata config.set_main_option( 'sqlalchemy.url', - str(current_app.extensions['migrate'].db.get_engine().url).replace( + str(current_app.extensions['migrate'].db.engine.url).replace( '%', '%%')) target_metadata = current_app.extensions['migrate'].db.metadata @@ -94,7 +94,7 @@ def process_revision_directives(context, revision, directives): directives[:] = [] logger.info('No changes in schema detected.') - connectable = current_app.extensions['migrate'].db.get_engine() + connectable = current_app.extensions['migrate'].db.engine with connectable.connect() as connection: context.configure( diff --git a/mslib/mscolab/migrations/script.py.mako b/mslib/mscolab/migrations/script.py.mako index 2c0156303..2943f763e 100644 --- a/mslib/mscolab/migrations/script.py.mako +++ b/mslib/mscolab/migrations/script.py.mako @@ -7,6 +7,7 @@ Create Date: ${create_date} """ from alembic import op import sqlalchemy as sa +import mslib.mscolab.custom_migration_types as cu ${imports if imports else ""} # revision identifiers, used by Alembic. diff --git a/mslib/mscolab/migrations/versions/83993fcdf5ef_.py b/mslib/mscolab/migrations/versions/83993fcdf5ef_.py deleted file mode 100644 index 1fa5d6c53..000000000 --- a/mslib/mscolab/migrations/versions/83993fcdf5ef_.py +++ /dev/null @@ -1,92 +0,0 @@ -# flake8: noqa -""" - -DB setup 6.x - -Revision ID: 83993fcdf5ef -Revises: cd8b7108713a -Create Date: 2021-10-04 09:22:36.987652 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '83993fcdf5ef' -down_revision = 'cd8b7108713a' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('operations', - sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('path', sa.String(length=255), nullable=True), - sa.Column('category', sa.String(length=255), nullable=True), - sa.Column('description', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('path') - ) - op.drop_table('projects') - with op.batch_alter_table('changes', schema=None) as batch_op: - batch_op.add_column(sa.Column('op_id', sa.Integer(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'operations', ['op_id'], ['id']) - batch_op.drop_column('p_id') - - with op.batch_alter_table('messages', schema=None) as batch_op: - batch_op.add_column(sa.Column('op_id', sa.Integer(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'operations', ['op_id'], ['id']) - batch_op.drop_column('p_id') - - with op.batch_alter_table('permissions', schema=None) as batch_op: - batch_op.add_column(sa.Column('op_id', sa.Integer(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'operations', ['op_id'], ['id']) - batch_op.drop_column('p_id') - - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.add_column(sa.Column('registered_on', sa.DateTime(), nullable=False)) - batch_op.add_column(sa.Column('confirmed', sa.Boolean(), nullable=False)) - batch_op.add_column(sa.Column('confirmed_on', sa.DateTime(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.drop_column('confirmed_on') - batch_op.drop_column('confirmed') - batch_op.drop_column('registered_on') - - with op.batch_alter_table('permissions', schema=None) as batch_op: - batch_op.add_column(sa.Column('p_id', sa.INTEGER(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'projects', ['p_id'], ['id']) - batch_op.drop_column('op_id') - - with op.batch_alter_table('messages', schema=None) as batch_op: - batch_op.add_column(sa.Column('p_id', sa.INTEGER(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'projects', ['p_id'], ['id']) - batch_op.drop_column('op_id') - - with op.batch_alter_table('changes', schema=None) as batch_op: - batch_op.add_column(sa.Column('p_id', sa.INTEGER(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'projects', ['p_id'], ['id']) - batch_op.drop_column('op_id') - - op.create_table('projects', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('path', sa.VARCHAR(length=255), nullable=True), - sa.Column('description', sa.VARCHAR(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('path') - ) - op.drop_table('operations') - # ### end Alembic commands ### diff --git a/mslib/mscolab/migrations/versions/922e4d9c94e2_to_version_10_0_0.py b/mslib/mscolab/migrations/versions/922e4d9c94e2_to_version_10_0_0.py new file mode 100644 index 000000000..c03710ee4 --- /dev/null +++ b/mslib/mscolab/migrations/versions/922e4d9c94e2_to_version_10_0_0.py @@ -0,0 +1,33 @@ +"""To version 10.0.0 + +Revision ID: 922e4d9c94e2 +Revises: c171019fe3ee +Create Date: 2024-07-24 15:28:42.009581 + +""" +from alembic import op +import sqlalchemy as sa +import mslib.mscolab.custom_migration_types as cu + + +# revision identifiers, used by Alembic. +revision = '922e4d9c94e2' +down_revision = 'c171019fe3ee' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('profile_image_path', sa.String(length=255), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_column('profile_image_path') + + # ### end Alembic commands ### diff --git a/mslib/mscolab/migrations/versions/cd8b7108713a_.py b/mslib/mscolab/migrations/versions/92eaba86a92e_to_version_8_3_5_initial_migration.py similarity index 52% rename from mslib/mscolab/migrations/versions/cd8b7108713a_.py rename to mslib/mscolab/migrations/versions/92eaba86a92e_to_version_8_3_5_initial_migration.py index 970b96816..ec675eab2 100644 --- a/mslib/mscolab/migrations/versions/cd8b7108713a_.py +++ b/mslib/mscolab/migrations/versions/92eaba86a92e_to_version_8_3_5_initial_migration.py @@ -1,18 +1,17 @@ -# flake8: noqa -""" -DB setup until v6.x +"""To version 8.3.5 - Initial migration -Revision ID: cd8b7108713a +Revision ID: 92eaba86a92e Revises: -Create Date: 2021-10-04 09:10:35.606793 +Create Date: 2024-07-08 15:47:24.916851 """ from alembic import op import sqlalchemy as sa +import mslib.mscolab.custom_migration_types as cu # revision identifiers, used by Alembic. -revision = 'cd8b7108713a' +revision = '92eaba86a92e' down_revision = None branch_labels = None depends_on = None @@ -20,55 +19,61 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('projects', + op.create_table('operations', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), sa.Column('path', sa.String(length=255), nullable=True), + sa.Column('category', sa.String(length=255), nullable=True), sa.Column('description', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('path') + sa.Column('active', sa.Boolean(), nullable=True), + sa.Column('last_used', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_operations')), + sa.UniqueConstraint('path', name=op.f('uq_operations_path')) ) op.create_table('users', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), sa.Column('username', sa.String(length=255), nullable=True), sa.Column('emailid', sa.String(length=255), nullable=True), sa.Column('password', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('emailid'), - sa.UniqueConstraint('password') + sa.Column('registered_on', sa.DateTime(), nullable=False), + sa.Column('confirmed', sa.Boolean(), nullable=False), + sa.Column('confirmed_on', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_users')), + sa.UniqueConstraint('emailid', name=op.f('uq_users_emailid')), + sa.UniqueConstraint('password', name=op.f('uq_users_password')) ) op.create_table('changes', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('p_id', sa.Integer(), nullable=True), + sa.Column('op_id', sa.Integer(), nullable=True), sa.Column('u_id', sa.Integer(), nullable=True), sa.Column('commit_hash', sa.String(length=255), nullable=True), sa.Column('version_name', sa.String(length=255), nullable=True), sa.Column('comment', sa.String(length=255), nullable=True), sa.Column('created_at', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['p_id'], ['projects.id'], ), - sa.ForeignKeyConstraint(['u_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') + sa.ForeignKeyConstraint(['op_id'], ['operations.id'], name=op.f('fk_changes_op_id_operations')), + sa.ForeignKeyConstraint(['u_id'], ['users.id'], name=op.f('fk_changes_u_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_changes')) ) op.create_table('messages', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('p_id', sa.Integer(), nullable=True), + sa.Column('op_id', sa.Integer(), nullable=True), sa.Column('u_id', sa.Integer(), nullable=True), sa.Column('text', sa.Text(), nullable=True), sa.Column('message_type', sa.Enum('TEXT', 'SYSTEM_MESSAGE', 'IMAGE', 'DOCUMENT', name='messagetype'), nullable=True), sa.Column('reply_id', sa.Integer(), nullable=True), sa.Column('created_at', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['p_id'], ['projects.id'], ), - sa.ForeignKeyConstraint(['reply_id'], ['messages.id'], ), - sa.ForeignKeyConstraint(['u_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') + sa.ForeignKeyConstraint(['op_id'], ['operations.id'], name=op.f('fk_messages_op_id_operations')), + sa.ForeignKeyConstraint(['reply_id'], ['messages.id'], name=op.f('fk_messages_reply_id_messages')), + sa.ForeignKeyConstraint(['u_id'], ['users.id'], name=op.f('fk_messages_u_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_messages')) ) op.create_table('permissions', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('p_id', sa.Integer(), nullable=True), + sa.Column('op_id', sa.Integer(), nullable=True), sa.Column('u_id', sa.Integer(), nullable=True), sa.Column('access_level', sa.Enum('admin', 'collaborator', 'viewer', 'creator', name='access_level'), nullable=True), - sa.ForeignKeyConstraint(['p_id'], ['projects.id'], ), - sa.ForeignKeyConstraint(['u_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') + sa.ForeignKeyConstraint(['op_id'], ['operations.id'], name=op.f('fk_permissions_op_id_operations')), + sa.ForeignKeyConstraint(['u_id'], ['users.id'], name=op.f('fk_permissions_u_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_permissions')) ) # ### end Alembic commands ### @@ -79,5 +84,5 @@ def downgrade(): op.drop_table('messages') op.drop_table('changes') op.drop_table('users') - op.drop_table('projects') + op.drop_table('operations') # ### end Alembic commands ### diff --git a/mslib/mscolab/migrations/versions/c171019fe3ee_to_version_9_0_0.py b/mslib/mscolab/migrations/versions/c171019fe3ee_to_version_9_0_0.py new file mode 100644 index 000000000..6b3e4ebda --- /dev/null +++ b/mslib/mscolab/migrations/versions/c171019fe3ee_to_version_9_0_0.py @@ -0,0 +1,35 @@ +"""To version 9.0.0 + +Revision ID: c171019fe3ee +Revises: 92eaba86a92e +Create Date: 2024-07-08 15:49:08.277483 + +""" +from alembic import op +import sqlalchemy as sa +import mslib.mscolab.custom_migration_types as cu + + +# revision identifiers, used by Alembic. +revision = 'c171019fe3ee' +down_revision = '92eaba86a92e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('authentication_backend', sa.String(length=255), nullable=False, default='local')) + batch_op.drop_constraint('uq_users_password', type_='unique') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.create_unique_constraint('uq_users_password', ['password']) + batch_op.drop_column('authentication_backend') + + # ### end Alembic commands ### diff --git a/mslib/mscolab/mscolab.py b/mslib/mscolab/mscolab.py index b0a79969d..445dc2f7c 100644 --- a/mslib/mscolab/mscolab.py +++ b/mslib/mscolab/mscolab.py @@ -34,11 +34,15 @@ import secrets import subprocess import git +import flask_migrate +import pathlib from mslib import __version__ +from mslib.mscolab import migrations from mslib.mscolab.conf import mscolab_settings from mslib.mscolab.seed import seed_data, add_user, add_all_users_default_operation, \ add_all_users_to_all_operations, delete_user +from mslib.mscolab.server import APP from mslib.mscolab.utils import create_files from mslib.utils import setup_logging from mslib.utils.qt import Worker, Updater @@ -65,24 +69,25 @@ def confirm_action(confirmation_prompt): print("Invalid input! Please select an option between y or n") -def handle_db_init(): - from mslib.mscolab.models import db - from mslib.mscolab.server import APP - create_files() - with APP.app_context(): - db.create_all() - print("Database initialised successfully!") - - def handle_db_reset(verbose=True): - from mslib.mscolab.models import db - from mslib.mscolab.server import APP - if os.path.exists(mscolab_settings.DATA_DIR): + if mscolab_settings.SQLALCHEMY_DB_URI.startswith("sqlite:///") and ( + db_path := pathlib.Path(mscolab_settings.SQLALCHEMY_DB_URI.removeprefix("sqlite:///")) + ).is_relative_to(mscolab_settings.DATA_DIR): + # Don't remove the database file + # This would be easier if the database wasn't stored in DATA_DIR... + p = pathlib.Path(mscolab_settings.DATA_DIR) + for root, dirs, files in os.walk(p, topdown=False): + for name in files: + full_file_path = pathlib.Path(root) / name + if full_file_path != db_path: + full_file_path.unlink() + for name in dirs: + (pathlib.Path(root) / name).rmdir() + elif os.path.exists(mscolab_settings.DATA_DIR): shutil.rmtree(mscolab_settings.DATA_DIR) create_files() - with APP.app_context(): - db.drop_all() - db.create_all() + flask_migrate.downgrade(directory=migrations.__path__[0], revision="base") + flask_migrate.upgrade(directory=migrations.__path__[0]) if verbose is True: print("Database has been reset successfully!") @@ -365,7 +370,6 @@ def main(): database_parser = subparsers.add_parser("db", help="Manage mscolab database") database_parser = database_parser.add_mutually_exclusive_group(required=True) - database_parser.add_argument("--init", help="Initialise database", action="store_true") database_parser.add_argument("--reset", help="Reset database", action="store_true") database_parser.add_argument("--seed", help="Seed database", action="store_true") database_parser.add_argument("--users_by_file", type=argparse.FileType('r'), @@ -417,18 +421,18 @@ def main(): handle_start(args) elif args.action == "db": - if args.init: - handle_db_init() - elif args.reset: + if args.reset: confirmation = confirm_action("Are you sure you want to reset the database? This would delete " "all your data! (y/[n]):") if confirmation is True: - handle_db_reset() + with APP.app_context(): + handle_db_reset() elif args.seed: confirmation = confirm_action("Are you sure you want to seed the database? Seeding will delete all your " "existing data and replace it with seed data (y/[n]):") if confirmation is True: - handle_db_seed() + with APP.app_context(): + handle_db_seed() elif args.users_by_file is not None: # fileformat: suggested_username name confirmation = confirm_action("Are you sure you want to add users to the database? (y/[n]):") diff --git a/mslib/mscolab/seed.py b/mslib/mscolab/seed.py index 366e221c5..8d9f23457 100644 --- a/mslib/mscolab/seed.py +++ b/mslib/mscolab/seed.py @@ -226,178 +226,175 @@ def archive_operation(path=None, emailid=None): def seed_data(): - app.config['SQLALCHEMY_DATABASE_URI'] = mscolab_settings.SQLALCHEMY_DB_URI - app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - with app.app_context(): - # create users - users = [{ - 'username': 'a', - 'id': 8, - 'password': 'a', - 'emailid': 'a@notexisting.org' - }, { - 'username': 'b', - 'id': 9, - 'password': 'b', - 'emailid': 'b@notexisting.org' - }, { - 'username': 'c', - 'id': 10, - 'password': 'c', - 'emailid': 'c@notexisting.org' - }, { - 'username': 'd', - 'id': 11, - 'password': 'd', - 'emailid': 'd@notexisting.org' - }, { - 'username': 'test1', - 'id': 12, - 'password': 'test1', - 'emailid': 'test1@notexisting.org' - }, { - 'username': 'test2', - 'id': 13, - 'password': 'test2', - 'emailid': 'test2@notexisting.org' - }, { - 'username': 'test3', - 'id': 14, - 'password': 'test3', - 'emailid': 'test3@notexisting.org' - }, { - 'username': 'test4', - 'id': 15, - 'password': 'test4', - 'emailid': 'test4@notexisting.org' - }, { - 'username': 'mscolab_user', - 'id': 16, - 'password': 'password', - 'emailid': 'mscolab_user@notexisting.org' - }, { - 'username': 'merge_waypoints_user', - 'id': 17, - 'password': 'password', - 'emailid': 'merge_waypoints_user@notexisting.org' - }] - for user in users: - db_user = User(user['emailid'], user['username'], user['password']) - db_user.id = user['id'] - db.session.add(db_user) - - # create operations - operations = [{ - 'id': 1, - 'path': 'one', - 'description': 'a, b', - 'category': 'default' - }, { - 'id': 2, - 'path': 'two', - 'description': 'b, c', - 'category': 'default' - }, { - 'id': 3, - 'path': 'three', - 'description': 'a, c', - 'category': 'default' - }, { - 'id': 4, - 'path': 'four', - 'description': 'd', - 'category': 'default' - }, { - 'id': 5, - 'path': 'Admin_Test', - 'description': 'Operation for testing admin window', - 'category': 'default' - }, { - 'id': 6, - 'path': 'test_mscolab', - 'description': 'Operation for testing mscolab main window', - 'category': 'default' - }] - for operation in operations: - db_operation = Operation(operation['path'], operation['description'], operation['category']) - db_operation.id = operation['id'] - db.session.add(db_operation) - - # create permissions - permissions = [{ - 'u_id': 8, - 'op_id': 1, - 'access_level': "creator" - }, { - 'u_id': 9, - 'op_id': 1, - 'access_level': "collaborator" - }, { - 'u_id': 9, - 'op_id': 2, - 'access_level': "creator" - }, { - 'u_id': 10, - 'op_id': 2, - 'access_level': "collaborator" - }, { - 'u_id': 10, - 'op_id': 3, - 'access_level': "creator" - }, { - 'u_id': 8, - 'op_id': 3, - 'access_level': "collaborator" - }, { - 'u_id': 10, - 'op_id': 1, - 'access_level': "viewer" - }, { - 'u_id': 11, - 'op_id': 4, - 'access_level': 'creator' - }, { - 'u_id': 8, - 'op_id': 4, - 'access_level': 'admin' - }, { - 'u_id': 13, - 'op_id': 3, - 'access_level': 'viewer' - }, { - 'u_id': 12, - 'op_id': 5, - 'access_level': 'creator' - }, { - 'u_id': 12, - 'op_id': 3, - 'access_level': 'collaborator' - }, { - 'u_id': 15, - 'op_id': 5, - 'access_level': 'viewer' - }, { - 'u_id': 14, - 'op_id': 3, - 'access_level': 'collaborator' - }, { - 'u_id': 15, - 'op_id': 3, - 'access_level': 'collaborator' - }, { - 'u_id': 16, - 'op_id': 6, - 'access_level': 'creator' - }, { - 'u_id': 17, - 'op_id': 6, - 'access_level': 'admin' - }] - for perm in permissions: - db_perm = Permission(perm['u_id'], perm['op_id'], perm['access_level']) - db.session.add(db_perm) - db.session.commit() - db.session.close() + # create users + users = [{ + 'username': 'a', + 'id': 8, + 'password': 'a', + 'emailid': 'a@notexisting.org' + }, { + 'username': 'b', + 'id': 9, + 'password': 'b', + 'emailid': 'b@notexisting.org' + }, { + 'username': 'c', + 'id': 10, + 'password': 'c', + 'emailid': 'c@notexisting.org' + }, { + 'username': 'd', + 'id': 11, + 'password': 'd', + 'emailid': 'd@notexisting.org' + }, { + 'username': 'test1', + 'id': 12, + 'password': 'test1', + 'emailid': 'test1@notexisting.org' + }, { + 'username': 'test2', + 'id': 13, + 'password': 'test2', + 'emailid': 'test2@notexisting.org' + }, { + 'username': 'test3', + 'id': 14, + 'password': 'test3', + 'emailid': 'test3@notexisting.org' + }, { + 'username': 'test4', + 'id': 15, + 'password': 'test4', + 'emailid': 'test4@notexisting.org' + }, { + 'username': 'mscolab_user', + 'id': 16, + 'password': 'password', + 'emailid': 'mscolab_user@notexisting.org' + }, { + 'username': 'merge_waypoints_user', + 'id': 17, + 'password': 'password', + 'emailid': 'merge_waypoints_user@notexisting.org' + }] + for user in users: + db_user = User(user['emailid'], user['username'], user['password']) + db_user.id = user['id'] + db.session.add(db_user) + + # create operations + operations = [{ + 'id': 1, + 'path': 'one', + 'description': 'a, b', + 'category': 'default' + }, { + 'id': 2, + 'path': 'two', + 'description': 'b, c', + 'category': 'default' + }, { + 'id': 3, + 'path': 'three', + 'description': 'a, c', + 'category': 'default' + }, { + 'id': 4, + 'path': 'four', + 'description': 'd', + 'category': 'default' + }, { + 'id': 5, + 'path': 'Admin_Test', + 'description': 'Operation for testing admin window', + 'category': 'default' + }, { + 'id': 6, + 'path': 'test_mscolab', + 'description': 'Operation for testing mscolab main window', + 'category': 'default' + }] + for operation in operations: + db_operation = Operation(operation['path'], operation['description'], operation['category']) + db_operation.id = operation['id'] + db.session.add(db_operation) + + # create permissions + permissions = [{ + 'u_id': 8, + 'op_id': 1, + 'access_level': "creator" + }, { + 'u_id': 9, + 'op_id': 1, + 'access_level': "collaborator" + }, { + 'u_id': 9, + 'op_id': 2, + 'access_level': "creator" + }, { + 'u_id': 10, + 'op_id': 2, + 'access_level': "collaborator" + }, { + 'u_id': 10, + 'op_id': 3, + 'access_level': "creator" + }, { + 'u_id': 8, + 'op_id': 3, + 'access_level': "collaborator" + }, { + 'u_id': 10, + 'op_id': 1, + 'access_level': "viewer" + }, { + 'u_id': 11, + 'op_id': 4, + 'access_level': 'creator' + }, { + 'u_id': 8, + 'op_id': 4, + 'access_level': 'admin' + }, { + 'u_id': 13, + 'op_id': 3, + 'access_level': 'viewer' + }, { + 'u_id': 12, + 'op_id': 5, + 'access_level': 'creator' + }, { + 'u_id': 12, + 'op_id': 3, + 'access_level': 'collaborator' + }, { + 'u_id': 15, + 'op_id': 5, + 'access_level': 'viewer' + }, { + 'u_id': 14, + 'op_id': 3, + 'access_level': 'collaborator' + }, { + 'u_id': 15, + 'op_id': 3, + 'access_level': 'collaborator' + }, { + 'u_id': 16, + 'op_id': 6, + 'access_level': 'creator' + }, { + 'u_id': 17, + 'op_id': 6, + 'access_level': 'admin' + }] + for perm in permissions: + db_perm = Permission(perm['u_id'], perm['op_id'], perm['access_level']) + db.session.add(db_perm) + db.session.commit() + db.session.close() with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as file_dir: file_paths = ['one', 'two', 'three', 'four', 'Admin_Test', 'test_mscolab'] diff --git a/mslib/mscolab/server.py b/mslib/mscolab/server.py index 90afff90b..c84c57f3a 100644 --- a/mslib/mscolab/server.py +++ b/mslib/mscolab/server.py @@ -34,6 +34,7 @@ import socketio import sqlalchemy.exc import werkzeug +import flask_migrate from itsdangerous import URLSafeTimedSerializer, BadSignature from flask import g, jsonify, request, render_template, flash @@ -53,9 +54,73 @@ from mslib.utils import conditional_decorator from mslib.index import create_app from mslib.mscolab.forms import ResetRequestForm, ResetPasswordForm +from mslib.mscolab import migrations + + +def _handle_db_upgrade(): + from mslib.mscolab.models import db + + create_files() + inspector = sqlalchemy.inspect(db.engine) + existing_tables = inspector.get_table_names() + if ("alembic_version" not in existing_tables and len(existing_tables) > 0) or ( + "alembic_version" in existing_tables + and len(existing_tables) > 1 + and db.session.execute(sqlalchemy.text("SELECT * FROM alembic_version")).first() is None + ): + sys.exit( + """Your database contains no alembic_version revision identifier, but it has a schema. This suggests \ +that you have a pre-existing database but haven't followed the database migration instructions. To prevent damage to \ +your database MSColab will abort. Please follow the documentation for a manual database migration from MSColab v8/v9.""" + ) + + is_empty_database = len(existing_tables) == 0 or ( + len(existing_tables) == 1 + and "alembic_version" in existing_tables + and db.session.execute(sqlalchemy.text("SELECT * FROM alembic_version")).first() is None + ) + # If a database connection to migrate from is set and the target database is empty, then migrate the existing data + if is_empty_database and mscolab_settings.SQLALCHEMY_DB_URI_TO_MIGRATE_FROM is not None: + logging.info("The target database is empty and a database to migrate from is set, starting the data migration") + source_engine = sqlalchemy.create_engine(mscolab_settings.SQLALCHEMY_DB_URI_TO_MIGRATE_FROM) + source_metadata = sqlalchemy.MetaData() + source_metadata.reflect(bind=source_engine) + # Determine the previous MSColab version based on the database content and upgrade to the corresponding revision + if "authentication_backend" in source_metadata.tables["users"].columns: + # It should be v9 + flask_migrate.upgrade(directory=migrations.__path__[0], revision="c171019fe3ee") + else: + # It's probably v8 + flask_migrate.upgrade(directory=migrations.__path__[0], revision="92eaba86a92e") + # Copy over the existing data + target_engine = sqlalchemy.create_engine(mscolab_settings.SQLALCHEMY_DB_URI) + target_metadata = sqlalchemy.MetaData() + target_metadata.reflect(bind=target_engine) + with source_engine.connect() as src_connection, target_engine.connect() as target_connection: + for table in source_metadata.sorted_tables: + if table.name == "alembic_version": + # Do not migrate the alembic_version table! + continue + logging.debug("Copying table %s", table.name) + stmt = target_metadata.tables[table.name].insert() + for row in src_connection.execute(table.select()): + logging.debug("Copying row %s", row) + row = tuple( + r.replace(tzinfo=datetime.timezone.utc) if isinstance(r, datetime.datetime) else r for r in row + ) + target_connection.execute(stmt.values(row)) + target_connection.commit() + logging.info("Data migration finished") + + # Upgrade to the latest database revision + flask_migrate.upgrade(directory=migrations.__path__[0]) + + logging.info("Database initialised successfully!") APP = create_app(__name__, imprint=mscolab_settings.IMPRINT, gdpr=mscolab_settings.GDPR) +with APP.app_context(): + _handle_db_upgrade() mail = Mail(APP) CORS(APP, origins=mscolab_settings.CORS_ORIGINS if hasattr(mscolab_settings, "CORS_ORIGINS") else ["*"]) auth = HTTPBasicAuth() diff --git a/setup.cfg b/setup.cfg index 147b23cfb..1c8297f6e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,6 @@ omit = norecursedirs = .git .idea .cache [flake8] -ignore = E124,E125,E402,W504,A005 +ignore = E124,E125,E402,W503,W504,A005 max-line-length = 120 exclude = mslib/msui/qt5/*.py, mslib/mscolab/migrations/*.py diff --git a/tests/_test_mscolab/test_migrations.py b/tests/_test_mscolab/test_migrations.py new file mode 100644 index 000000000..ae0b7fba4 --- /dev/null +++ b/tests/_test_mscolab/test_migrations.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +""" + + tests._test_mscolab.test_migrations + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Tests for database migrations. + + This file is part of MSS. + + :copyright: Copyright 2024 Matthias Riße + :copyright: Copyright 2024 by the MSS team, see AUTHORS. + :license: APACHE-2.0, see LICENSE for details. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import pytest +import itertools +import flask +import flask_migrate +import sqlalchemy +import mslib.mscolab.migrations +import mslib.index +from mslib.mscolab.app import db, migrate +from mslib.mscolab.conf import mscolab_settings + + +def test_migrations(mscolab_app): + migrations_path = mslib.mscolab.migrations.__path__[0] + with mscolab_app.app_context(): + # Seed the database and try to downgrade to each previous revision and then upgrade to the latest again + mslib.mscolab.mscolab.handle_db_seed() + backward_steps = 1 + all_revisions_tested = False + while not all_revisions_tested: + for _ in range(backward_steps): + try: + flask_migrate.downgrade(directory=migrations_path) + except SystemExit as e: + if e.code == 1: + all_revisions_tested = True + flask_migrate.upgrade(directory=migrations_path) + backward_steps += 1 + # Check that there are no differences between the now-current database schema and the defined model + try: + flask_migrate.check(directory=migrations_path) + except SystemExit as e: + assert ( + e.code == 0 + ), "The database models are inconsistent with the migration scripts. Did you forget to add a migration?" + + +_revision_to_name = { + "92eaba86a92e": "v8", + "c171019fe3ee": "v9", +} + +_cases = list( + pytest.param(revision, iterations, id=f"{_revision_to_name[revision]}-iterations={iterations}") + for revision, iterations in itertools.product(["92eaba86a92e", "c171019fe3ee"], [1, 2, 100]) +) + + +@pytest.mark.parametrize("revision,iterations", _cases) +def test_upgrade_from(revision, iterations, mscolab_app, tmp_path): + """Test upgrading from a pre-v10 database that wasn't yet automatically managed with flask-migrate.""" + migrations_path = mslib.mscolab.migrations.__path__[0] + # Construct a dummy flask app to create a separate database to migrate from + # TODO: this would be easier if it was possible to create multiple canonical MSColab Flask app instances, + # i.e. if there was a factory function instead of one global instance. This test could then check the correct + # functioning of the data migration while creating such an app instance, instead of having to call a private method. + app = flask.Flask("whatever") + # TODO: make this somehow configurable to use something other than sqlite as the source database + app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///" + str(tmp_path.absolute() / "mscolab.db") + db.init_app(app) + migrate.init_app(app, db) + with app.app_context(): + # Seed the database and downgrade to the supplied revision + mslib.mscolab.mscolab.handle_db_seed() + flask_migrate.downgrade(directory=migrations_path, revision=revision) + # Set the alembic_version to a non-existing revision to simulate a manual migration following the old migration + # instructions. + db.session.execute(sqlalchemy.text("UPDATE alembic_version SET version_num = 'e62d08ce88a4'")) + # Collect all data for comparison with what's copied over + metadata = sqlalchemy.MetaData() + metadata.reflect(bind=db.engine) + expected_data = {name: db.session.execute(table.select()).all() for name, table in metadata.tables.items()} + del expected_data["alembic_version"] # the alembic_version table will be different, but that is expected + + try: + mscolab_settings.SQLALCHEMY_DB_URI_TO_MIGRATE_FROM = app.config["SQLALCHEMY_DATABASE_URI"] + with mscolab_app.app_context(): + db.drop_all() + db.session.execute(sqlalchemy.text("DROP TABLE alembic_version")) + inspector = sqlalchemy.inspect(db.engine) + existing_tables = inspector.get_table_names() + assert existing_tables == [] + + # Also try multiple applications of the db upgrade to ensure idempotence of the operation + for _ in range(iterations): + mslib.mscolab.server._handle_db_upgrade() + + # Check that no further migration is required + flask_migrate.check(directory=migrations_path) + actual_data = {name: db.session.execute(table.select()).all() for name, table in db.metadata.tables.items()} + # Check that all tables have the right number of entries with matching ids copied over + assert {k: [e[0] for e in v] for k, v in expected_data.items()} == { + k: [e[0] for e in v] for k, v in actual_data.items() + } + # TODO: Maybe add more asserts? Basically anything could break with future migrations though, if the schema + # is fundamentally changed. Having an id as the first column is already an assumption that might not always + # hold (but should be reliable enough). + flask_migrate.downgrade(directory=migrations_path, revision=revision) + metadata = sqlalchemy.MetaData() + metadata.reflect(bind=db.engine) + actual_data_after_downgrade = { + name: db.session.execute(table.select()).all() for name, table in metadata.tables.items() + } + del actual_data_after_downgrade["alembic_version"] # expected data doesn't have the revision table + # Check that after a downgrade the data is definitely the same + assert expected_data == actual_data_after_downgrade + finally: + mscolab_settings.SQLALCHEMY_DB_URI_TO_MIGRATE_FROM = None diff --git a/tests/_test_mscolab/test_mscolab.py b/tests/_test_mscolab/test_mscolab.py index 6e6882d39..ca395d9f6 100644 --- a/tests/_test_mscolab/test_mscolab.py +++ b/tests/_test_mscolab/test_mscolab.py @@ -53,7 +53,7 @@ def test_main(): with mock.patch("mslib.mscolab.mscolab.argparse.ArgumentParser.parse_args", return_value=argparse.Namespace(version=False, update=False, action="db", - init=False, reset=False, seed=False, users_by_file=None, + reset=False, seed=False, users_by_file=None, default_operation=False, add_all_to_all_operation=False, delete_users_by_file=False)): main() diff --git a/tests/fixtures.py b/tests/fixtures.py index dfd29a39e..0a4e03f9c 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -37,7 +37,7 @@ from contextlib import contextmanager from mslib.mscolab.conf import mscolab_settings from mslib.mscolab.server import APP, sockio, cm, fm -from mslib.mscolab.mscolab import handle_db_init, handle_db_reset +from mslib.mscolab.mscolab import handle_db_reset from mslib.utils.config import modify_config_file from tests.utils import is_url_response_ok @@ -94,7 +94,6 @@ def mscolab_session_app(): _app.config['SQLALCHEMY_DATABASE_URI'] = mscolab_settings.SQLALCHEMY_DB_URI _app.config['MSCOLAB_DATA_DIR'] = mscolab_settings.MSCOLAB_DATA_DIR _app.config['UPLOAD_FOLDER'] = mscolab_settings.UPLOAD_FOLDER - handle_db_init() return _app @@ -138,7 +137,8 @@ def reset_mscolab(mscolab_session_app): This fixture is not explicitly needed in tests, it is used in the other fixtures to do the cleanup actions. """ - handle_db_reset() + with mscolab_session_app.app_context(): + handle_db_reset() @pytest.fixture