Migrations for new data schema

open-craft · May 18, 2016 · 0b4c63e · e-kolpakov · May 25, 2016 · bradenmacdonald
1 parent 05a797e
commit 0b4c63e
Show file tree

Hide file tree

Showing 8 changed files with 351 additions and 17 deletions.
diff --git a/instance/migrations/0035_reset_ansible_settings.py b/instance/migrations/0035_reset_ansible_settings.py
@@ -1,23 +1,33 @@
 # -*- coding: utf-8 -*-
 import sys
 from django.db import migrations, models
-from instance.models.instance import SingleVMOpenEdXInstance as CurrentOpenEdXInstance
+"""
+Generate ansible_settings:
 
+This data migration is no longer supported. All existing installations have this migration
+applied, and new installations do not depend on data migrations.
 
-def reset_ansible_settings(apps, schema_editor):
-    db_alias = schema_editor.connection.alias
-    HistoricalOpenEdXInstance = apps.get_model("instance", "OpenEdXInstance")
-    for instance in HistoricalOpenEdXInstance.objects.using(db_alias).iterator():
-        if not instance.ansible_settings:
-            try:
-                # Use the current version of OpenEdXInstance instead of the historical version from
-                # the app registry, since the faked version from the registry doesn't have any of
-                # the custom fields and methods.  I don't think there is any better way to do this.
-                current_instance = CurrentOpenEdXInstance.objects.using(db_alias).get(pk=instance.pk)
-                current_instance.reset_ansible_settings(commit=True)
-            except Exception as exc:
-                print('Error while migrating {}: {}'.format(instance, exc), file=sys.stderr)
-                print('Ignoring error and carrying on.', file=sys.stderr)
+We had to disable it since the 'import SingleVMOpenEdXInstance as CurrentOpenEdXInstance'
+hack no longer works now that that model's code has been split up among other models.
+"""
+
+# from instance.models.instance import SingleVMOpenEdXInstance as CurrentOpenEdXInstance
+
+
+# def reset_ansible_settings(apps, schema_editor):
+#     db_alias = schema_editor.connection.alias
+#     HistoricalOpenEdXInstance = apps.get_model("instance", "OpenEdXInstance")
+#     for instance in HistoricalOpenEdXInstance.objects.using(db_alias).iterator():
+#         if not instance.ansible_settings:
+#             try:
+#                 # Use the current version of OpenEdXInstance instead of the historical version from
+#                 # the app registry, since the faked version from the registry doesn't have any of
+#                 # the custom fields and methods.  I don't think there is any better way to do this.
+#                 current_instance = CurrentOpenEdXInstance.objects.using(db_alias).get(pk=instance.pk)
+#                 current_instance.reset_ansible_settings(commit=True)
+#             except Exception as exc:
+#                 print('Error while migrating {}: {}'.format(instance, exc), file=sys.stderr)
+#                 print('Ignoring error and carrying on.', file=sys.stderr)
 
 
 class Migration(migrations.Migration):
@@ -27,5 +37,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(reset_ansible_settings, migrations.RunPython.noop),
+        #migrations.RunPython(reset_ansible_settings, migrations.RunPython.noop),
+        migrations.RunPython(migrations.RunPython.noop, migrations.RunPython.noop),
     ]
diff --git a/instance/migrations/0042_add_instance_status.py b/instance/migrations/0042_add_instance_status.py
@@ -3,7 +3,22 @@
 
 from django.db import migrations, models
 
-from instance.models.instance import Status as InstanceStatus
+class InstanceStatus:
+    """ Frozen copy of InstanceStatus constants at this point in the migration history """
+    class New:
+        state_id = 'new'
+    class WaitingForServer:
+        state_id = 'waiting'
+    class ConfiguringServer:
+        state_id = 'configuring'
+    class Running:
+        state_id = 'running'
+    class ConfigurationFailed:
+        state_id = 'failed'
+    class Error:
+        state_id = 'error'
+    class Terminated:
+        state_id = 'terminated'
 
 
 def get_current_server(instance):

diff --git a/instance/migrations/0048_appserver_refactor1.py b/instance/migrations/0048_appserver_refactor1.py
@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import instance.models.instance
+import instance.models.mixins.utilities
+import instance.models.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('instance', '0047_generic_logging3'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='InstanceReference',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('name', models.CharField(default='Instance', max_length=250)),
+                ('instance_id', models.PositiveIntegerField()),
+                ('instance_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType')),
+            ],
+            options={
+                'ordering': ['-created'],
+            },
+        ),
+        migrations.CreateModel(
+            name='OpenEdXAppServer',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('_status', models.CharField(choices=[('configuring', 'ConfiguringServer'), ('error', 'Error'), ('failed', 'ConfigurationFailed'), ('new', 'New'), ('running', 'Running'), ('terminated', 'Terminated'), ('waiting', 'WaitingForServer')], db_column='status', db_index=True, default='new', max_length=20)),
+                ('name', models.CharField(max_length=250)),
+                ('email', models.EmailField(default='contact@example.com', help_text='The default contact email for this instance; also used as the from address for emails sent by the server.', max_length=254)),
+                ('protocol', models.CharField(choices=[('http', 'HTTP - Unencrypted clear text'), ('https', 'HTTPS - Encrypted')], default='http', max_length=5)),
+                ('configuration_source_repo_url', models.URLField(max_length=256)),
+                ('configuration_version', models.CharField(max_length=50)),
+                ('configuration_extra_settings', models.TextField(blank=True, help_text='YAML config vars that override all others')),
+                ('edx_platform_repository_url', models.CharField(help_text='URL to the edx-platform repository to use. Leave blank for default.', max_length=256)),
+                ('edx_platform_commit', models.CharField(help_text='edx-platform commit hash or branch or tag to use. Leave blank to use the default, which is equal to the value of "openedx_release".', max_length=256)),
+                ('openedx_release', models.CharField(help_text='Set this to a release tag like "named-release/dogwood" to build a specific release of Open edX. This setting becomes the default value for edx_platform_version, forum_version, notifier_version, xqueue_version, and certs_version so it should be a git branch that exists in all of those repositories. Note: to build a specific branch of edx-platform, you should just override edx_platform_commit rather than changing this setting. Note 2: This value does not affect the default value of configuration_version.', max_length=128)),
+                ('use_ephemeral_databases', models.BooleanField()),
+                ('github_admin_organization_name', models.CharField(blank=True, default='', help_text="GitHub organization whose users will be given SSH access to this instance's VMs", max_length=200)),
+                ('configuration_database_settings', models.TextField(blank=True, help_text='YAML vars for database configuration')),
+                ('configuration_storage_settings', models.TextField(blank=True, help_text='YAML vars for storage configuration')),
+                ('configuration_settings', models.TextField(help_text='A record of the combined (final) ansible variables passed to the configuration playbook when configuring this AppServer.')),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='openedxappserver_set', to='instance.InstanceReference')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(instance.models.utils.ValidateModelMixin, models.Model, instance.models.mixins.utilities.EmailMixin),
+        ),
+        migrations.CreateModel(
+            name='OpenEdXInstance',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('mysql_user', models.CharField(blank=True, max_length=16)),
+                ('mysql_pass', models.CharField(blank=True, max_length=32)),
+                ('mysql_provisioned', models.BooleanField(default=False)),
+                ('mongo_user', models.CharField(blank=True, max_length=16)),
+                ('mongo_pass', models.CharField(blank=True, max_length=32)),
+                ('mongo_provisioned', models.BooleanField(default=False)),
+                ('swift_openstack_user', models.CharField(blank=True, max_length=32)),
+                ('swift_openstack_password', models.CharField(blank=True, max_length=64)),
+                ('swift_openstack_tenant', models.CharField(blank=True, max_length=32)),
+                ('swift_openstack_auth_url', models.URLField(blank=True)),
+                ('swift_openstack_region', models.CharField(blank=True, max_length=16)),
+                ('swift_provisioned', models.BooleanField(default=False)),
+                ('email', models.EmailField(default='contact@example.com', help_text='The default contact email for this instance; also used as the from address for emails sent by the server.', max_length=254)),
+                ('protocol', models.CharField(choices=[('http', 'HTTP - Unencrypted clear text'), ('https', 'HTTPS - Encrypted')], default='http', max_length=5)),
+                ('configuration_source_repo_url', models.URLField(max_length=256)),
+                ('configuration_version', models.CharField(max_length=50)),
+                ('configuration_extra_settings', models.TextField(blank=True, help_text='YAML config vars that override all others')),
+                ('edx_platform_repository_url', models.CharField(help_text='URL to the edx-platform repository to use. Leave blank for default.', max_length=256)),
+                ('edx_platform_commit', models.CharField(help_text='edx-platform commit hash or branch or tag to use. Leave blank to use the default, which is equal to the value of "openedx_release".', max_length=256)),
+                ('openedx_release', models.CharField(help_text='Set this to a release tag like "named-release/dogwood" to build a specific release of Open edX. This setting becomes the default value for edx_platform_version, forum_version, notifier_version, xqueue_version, and certs_version so it should be a git branch that exists in all of those repositories. Note: to build a specific branch of edx-platform, you should just override edx_platform_commit rather than changing this setting. Note 2: This value does not affect the default value of configuration_version.', max_length=128)),
+                ('s3_access_key', models.CharField(blank=True, max_length=50)),
+                ('s3_secret_access_key', models.CharField(blank=True, max_length=50)),
+                ('s3_bucket_name', models.CharField(blank=True, max_length=50)),
+                ('use_ephemeral_databases', models.BooleanField()),
+                ('github_admin_organization_name', models.CharField(blank=True, default='', help_text="GitHub organization whose users will be given SSH access to this instance's VMs", max_length=200)),
+                ('sub_domain', models.CharField(max_length=50)),
+                ('base_domain', models.CharField(blank=True, max_length=50)),
+                ('active_appserver', models.OneToOneField(null=True, blank=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='instance.OpenEdXAppServer')),
+            ],
+            options={
+                'verbose_name': 'Open edX Instance',
+            },
+            bases=(instance.models.instance.Instance, models.Model),
+        ),
+        migrations.AddField(
+            model_name='openstackserver',
+            name='name_prefix',
+            field=models.SlugField(default='edxapp-old', max_length=20),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='openedxappserver',
+            name='server',
+            field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='instance.OpenStackServer'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='openedxinstance',
+            unique_together=set([('base_domain', 'sub_domain')]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='instancereference',
+            unique_together=set([('instance_type', 'instance_id')]),
+        ),
+    ]
diff --git a/instance/migrations/0049_appserver_refactor2.py b/instance/migrations/0049_appserver_refactor2.py
@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import instance.models.instance
+import instance.models.mixins.utilities
+import instance.models.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('instance', '0048_appserver_refactor1'),
+        ('pr_watch', '0001_initial'),
+    ]
+
+    def data_forward(apps, schema_editor):
+        """
+        Move existing data from SingleVMOpenEdXInstance to OpenEdXInstance + OpenEdXAppServer
+        """
+        # Unchanged models:
+        ContentType = apps.get_model('contenttypes', 'ContentType')
+        OpenStackServer = apps.get_model('instance', 'OpenStackServer')
+        # Old models:
+        SingleVMOpenEdXInstance = apps.get_model('instance', 'SingleVMOpenEdXInstance')
+        # New models:
+        OpenEdXInstance = apps.get_model('instance', 'OpenEdXInstance')
+        OpenEdXAppServer = apps.get_model('instance', 'OpenEdXAppServer')
+        WatchedPullRequest = apps.get_model('pr_watch', 'WatchedPullRequest')
+
+        for old_instance in SingleVMOpenEdXInstance.objects.all().order_by('pk'):
+            # Compute the 'repository_url' property of old_instance since we need it but it's not directly available:
+            repository_url = 'https://github.com/{}/{}.git'.format(
+                old_instance.github_organization_name, old_instance.github_repository_name,
+            )
+
+            # Create the new OpenEdXInstance:
+            new_instance = OpenEdXInstance.objects.create(
+                # Main:
+                name=old_instance.name,
+                email=old_instance.email,
+                protocol=old_instance.protocol,
+                use_ephemeral_databases=old_instance.use_ephemeral_databases,
+                github_admin_organization_name=old_instance.github_admin_organization_name,
+                sub_domain=old_instance.sub_domain,
+                base_domain=old_instance.base_domain,
+                active_appserver=None,
+                # Database:
+                mysql_user=old_instance.mysql_user,
+                mysql_pass=old_instance.mysql_pass,
+                mysql_provisioned=old_instance.mysql_provisioned,
+                mongo_user=old_instance.mongo_user,
+                mongo_pass=old_instance.mongo_pass,
+                mongo_provisioned=old_instance.mongo_provisioned,
+                # Storage:
+                swift_openstack_user=old_instance.swift_openstack_user,
+                swift_openstack_password=old_instance.swift_openstack_password,
+                swift_openstack_tenant=old_instance.swift_openstack_tenant,
+                swift_openstack_auth_url=old_instance.swift_openstack_auth_url,
+                swift_openstack_region=old_instance.swift_openstack_region,
+                swift_provisioned=old_instance.swift_provisioned,
+                s3_access_key=old_instance.s3_access_key,
+                s3_secret_access_key=old_instance.s3_secret_access_key,
+                s3_bucket_name=old_instance.s3_bucket_name,
+                # Ansible:
+                configuration_source_repo_url=old_instance.ansible_source_repo_url,
+                configuration_version=old_instance.configuration_version,
+                configuration_extra_settings=old_instance.ansible_extra_settings,
+                edx_platform_repository_url=repository_url,
+                edx_platform_commit=old_instance.commit_id,
+                # openedx_release has replaced forum_version, notifier_version, etc. which were usually the same:
+                openedx_release=old_instance.forum_version,
+            )
+
+            # Create the PR record, if any:
+            if old_instance.github_pr_url:
+                watched_pr = WatchedPullRequest.objects.create(
+                    branch_name=old_instance.branch_name,
+                    ref_type=old_instance.ref_type,
+                    github_organization_name=old_instance.github_organization_name,
+                    github_repository_name=old_instance.github_repository_name,
+                    github_pr_url=old_instance.github_pr_url,
+                    instance=new_instance,
+                )
+
+            # Get the VM, if any:
+            current_vm = old_instance.server_set.order_by("id").last()
+            if current_vm:
+                # Create one AppServer:
+                new_appserver = OpenEdXAppServer.objects.create(
+                    # Relations to other objects:
+                    owner_id=new_instance.ref.pk,
+                    server=current_vm,
+                    # Timestamps:
+                    created=old_instance.created,
+                    modified=old_instance.modified,
+                    # Basic properties:
+                    name="AppServer 1",
+                    _status=old_instance._status,
+                    # Main ansible var settings:
+                    email=old_instance.email,
+                    protocol=old_instance.protocol,
+                    configuration_source_repo_url=old_instance.ansible_source_repo_url,
+                    configuration_version=old_instance.configuration_version,
+                    configuration_extra_settings=old_instance.ansible_extra_settings,
+                    edx_platform_repository_url=repository_url,
+                    edx_platform_commit=old_instance.commit_id,
+                    openedx_release=new_instance.openedx_release,
+                    use_ephemeral_databases=old_instance.use_ephemeral_databases,
+                    github_admin_organization_name=old_instance.github_admin_organization_name,
+                    # Derived ansible var YAML:
+                    configuration_database_settings="",
+                    configuration_storage_settings="",
+                    configuration_settings=old_instance.ansible_settings,
+                )
+                if old_instance._status == 'running':
+                    new_instance.active_appserver = new_appserver
+                    new_instance.save()
+
+            # Migrate LogEntry entries:
+            LogEntry = apps.get_model('instance', 'logentry')
+            old_instance_type = ContentType.objects.get_for_model(SingleVMOpenEdXInstance)
+            new_instance_type = ContentType.objects.get_for_model(OpenEdXInstance)
+            LogEntry.objects.filter(content_type=old_instance_type).update(content_type=new_instance_type)
+
+    def data_backward(apps, schema_editor):
+        """
+        Reverse the data migration
+        """
+        raise NotImplementedError
+
+    operations = [
+        migrations.RunPython(data_forward, data_backward),
+    ]