diff --git a/instance/logger_adapter.py b/instance/logger_adapter.py index ed46fb3ab..684cf3874 100644 --- a/instance/logger_adapter.py +++ b/instance/logger_adapter.py @@ -24,10 +24,33 @@ import logging +# Helper methods ############################################################## + + +def format_instance(instance): + """ Given any concrete subclass of Instance, get a short ID string to put into the log """ + if instance: + return 'instance={} ({!s:.15})'.format(instance.ref.pk, instance.ref.name) + return 'Unknown Instance' + + +def format_appserver(app_server): + """ Given an AppServer subclass, get a short ID string to put into the log """ + if app_server: + return 'app_server={} ({!s:.15})'.format(app_server.pk, app_server.name) + return 'Unknown AppServer' + + +def format_server(server): + """ Given a Server subclass, get a short ID string to put into the log """ + if server: + return 'server={!s:.20}'.format(server.name) + return 'Unknown Server' # Adapters #################################################################### -class InstanceLoggerAdapter(logging.LoggerAdapter): + +class AppServerLoggerAdapter(logging.LoggerAdapter): """ Custom LoggerAdapter for Instance objects Include the instance name in the output @@ -35,12 +58,16 @@ class InstanceLoggerAdapter(logging.LoggerAdapter): def process(self, msg, kwargs): msg, kwargs = super().process(msg, kwargs) - instance = self.extra['obj'] - if instance.sub_domain: - return 'instance={} | {}'.format(instance.sub_domain, msg), kwargs + app_server = self.extra['obj'] + if app_server.sub_domain: + return 'app_server={} | {}'.format(app_server.name, msg), kwargs else: return msg, kwargs + app_server = self.extra['obj'] + msg = '{},{} | {}'.format(format_instance(app_server.instance), format_appserver(app_server), msg) + return msg, kwargs + class ServerLoggerAdapter(logging.LoggerAdapter): """ @@ -51,7 +78,18 @@ def process(self, msg, kwargs): msg, kwargs = super().process(msg, kwargs) server = self.extra['obj'] - if server.instance and server.instance.sub_domain: - return 'instance={!s:.15},server={!s:.8} | {}'.format(server.instance.sub_domain, server, msg), kwargs - else: - return msg, kwargs + msg = '{} | {}'.format(format_server(server), msg) + return msg, kwargs + + +class InstanceLoggerAdapter(logging.LoggerAdapter): + """ + Custom LoggerAdapter for Instance objects + Include the InstanceReference ID in the output + """ + def process(self, msg, kwargs): + msg, kwargs = super().process(msg, kwargs) + + instance = self.extra['obj'] + msg = '{} | {}'.format(format_instance(instance), msg) + return msg, kwargs diff --git a/instance/models/appserver.py b/instance/models/appserver.py new file mode 100644 index 000000000..a5700d1c9 --- /dev/null +++ b/instance/models/appserver.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# +# OpenCraft -- tools to aid developing and hosting free software projects +# Copyright (C) 2015 OpenCraft +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +""" +Instance app models - App Server +""" + +# Imports ##################################################################### + +import logging + +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.db import models +from django.db.models import Q +from django_extensions.db.models import TimeStampedModel + +from instance.logger_adapter import AppServerLoggerAdapter +from .instance import InstanceReference +from .log_entry import LogEntry +from .server import OpenStackServer +from .utils import ModelResourceStateDescriptor, ResourceState, ValidateModelMixin + + +# Logging ##################################################################### + +logger = logging.getLogger(__name__) + + +# States ###################################################################### + +class AppServerState(ResourceState): + """ + A [finite state machine] state describing an instance. + """ + # A state is "steady" if we don't expect it to change. + # This information can be used to: + # - delay execution of an operation until the target server reaches a steady state + # - raise an exception when trying to schedule an operation that depends on a state change + # while the target server is in a steady state + # Steady states include: + # - Status.New + # - Status.Running + # - Status.ConfigurationFailed + # - Status.Error + # - Status.Terminated + is_steady_state = True + + # An instance is healthy if it is part of a normal (expected) workflow. + # This information can be used to detect problems and highlight them in the UI or notify users. + # Healthy states include: + # - Status.New + # - Status.WaitingForServer + # - Status.ConfiguringServer + # - Status.Running + # - Status.Terminated + is_healthy_state = True + + +class Status(ResourceState.Enum): + """ + The states that an instance can be in. + """ + + class New(AppServerState): + """ Newly created """ + state_id = 'new' + + class WaitingForServer(AppServerState): + """ Server not yet accessible """ + state_id = 'waiting' + name = 'Waiting for server' + is_steady_state = False + + class ConfiguringServer(AppServerState): + """ Running Ansible playbooks on server """ + state_id = 'configuring' + name = 'Configuring server' + is_steady_state = False + + class Running(AppServerState): + """ Instance is up and running """ + state_id = 'running' + + class ConfigurationFailed(AppServerState): + """ Instance was not configured successfully (but may be partially online) """ + state_id = 'failed' + name = 'Configuration failed' + is_healthy_state = False + + class Error(AppServerState): + """ Instance never got up and running (something went wrong when trying to build new VM) """ + state_id = 'error' + is_healthy_state = False + + class Terminated(AppServerState): + """ Instance was running successfully and has been shut down """ + state_id = 'terminated' + + +# Models ###################################################################### + + +class AppServer(ValidateModelMixin, TimeStampedModel): + """ + AppServer - One or more distinct web applications running on a single VM. + + Owned by an Instance. + + Characteristics of an AppServer: + * An AppServer instance's configuration fields are *immutable*. If you want to change + configuration, change hte Instance's configuration and have it create a new AppServer. + * An AppServer owns exactly one VM (Server), onto which it installs its applications. + """ + Status = Status + status = ModelResourceStateDescriptor( + state_classes=Status.states, default_state=Status.New, model_field_name='_status' + ) + _status = models.CharField( + max_length=20, + default=status.default_state_class.state_id, + choices=status.model_field_choices, + db_index=True, + db_column='status', + ) + # State transitions: + _status_to_waiting_for_server = status.transition( + from_states=Status.New, to_state=Status.WaitingForServer + ) + _status_to_configuring_server = status.transition( + from_states=Status.WaitingForServer, to_state=Status.ConfiguringServer + ) + _status_to_error = status.transition( + from_states=Status.WaitingForServer, to_state=Status.Error + ) + _status_to_running = status.transition( + from_states=Status.ConfiguringServer, to_state=Status.Running + ) + _status_to_configuration_failed = status.transition( + from_states=Status.ConfiguringServer, to_state=Status.ConfigurationFailed + ) + _status_to_terminated = status.transition( + from_states=Status.Running, to_state=Status.Terminated + ) + + name = models.CharField(max_length=250, blank=False) + server = models.OneToOneField(OpenStackServer, on_delete=models.CASCADE, related_name='+') + # The Instance that owns this. Instance will get related_name accessors like 'openedxappserver_set' + owner = models.ForeignKey(InstanceReference, on_delete=models.CASCADE, related_name='%(class)s_set') + + class Meta: + abstract = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.logger = AppServerLoggerAdapter(logger, {'obj': self}) + + def __str__(self): + return self.name + + @property + def instance(self): + """ + Get the Instance that owns this AppServer + """ + return self.owner.instance + + @property + def event_context(self): + """ + Context dictionary to include in events + """ + context = self.instance.event_context # dict with instance_id + context.update({'appserver_id': self.pk, 'appserver_type': self.__class__.__name__}) + return context + + def set_field_defaults(self): + """ + Set default values. + + We don't use django field 'defaults=callable' because changes to the field defaults + affect automatically generated migrations, generating a new one when settings don't match + + This method is called by clean_fields() before validation and save() + """ + pass + + def clean_fields(self, exclude=None): + """ + Set default values before validation. + """ + if not self.pk: + self.set_field_defaults() + super().clean_fields(exclude=exclude) + + def save(self, **kwargs): + if self.pk: + # We are changing an existing instance. But most AppServer fields are meant to be + # immutable. Only 'status' can change. + if not set(kwargs.get('update_fields', [])) < set(['_status', 'modified']): + raise RuntimeError("Error: Attempted to modify an AppServer instance. AppServers are immutable.") + super().save(**kwargs) + + def _get_log_entries(self, level_list=None, limit=None): + """ + Return the list of log entry instances for this AppServer and the server it manages, + optionally filtering by logging level. If a limit is given, only the latest records are + returned. + + Returns oldest entries first. + """ + # TODO: Filter out log entries for which the user doesn't have view rights + appserver_type = ContentType.objects.get_for_model(self) + server_type = ContentType.objects.get_for_model(self.server) + entries = LogEntry.objects.filter( + (Q(content_type=appserver_type) & Q(object_id=self.pk)) | + (Q(content_type=server_type) & Q(object_id=self.server_id)) + ) + if level_list: + entries = entries.filter(level__in=level_list) + if limit: + return reversed(list(entries[:limit])) + return entries.order_by('created') + + @property + def log_entries(self): + """ + Return the list of log entry instances for this AppServer and the server it manages + """ + return self._get_log_entries(limit=settings.LOG_LIMIT) + + @property + def log_error_entries(self): + """ + Return the list of error or critical log entry instances for this AppServer and the + server it manages + """ + return self._get_log_entries(level_list=['ERROR', 'CRITICAL']) diff --git a/instance/models/instance.py b/instance/models/instance.py index ce425423f..2684c2019 100644 --- a/instance/models/instance.py +++ b/instance/models/instance.py @@ -17,44 +17,18 @@ # along with this program. If not, see . # """ -Instance app models - Instance +Instance app models - Open EdX Instance and AppServer models """ - -# Imports ##################################################################### - import logging -import string -from django.conf import settings +from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.contrib.contenttypes.models import ContentType from django.db import models -from django.db.models import Q -from django.db.backends.utils import truncate_name -from django.template import loader -from django.utils import timezone -from django.utils.crypto import get_random_string +from django.utils.functional import cached_property from django_extensions.db.models import TimeStampedModel -from instance.gandi import GandiAPI from instance.logger_adapter import InstanceLoggerAdapter -from instance.logging import log_exception -from instance.models.log_entry import LogEntry -from instance.models.mixins.ansible import AnsibleInstanceMixin -from instance.models.mixins.database import MongoDBInstanceMixin, MySQLInstanceMixin, SwiftContainerInstanceMixin -from instance.models.mixins.utilities import EmailInstanceMixin -from instance.models.mixins.version_control import GitHubInstanceMixin -from instance.models.utils import ( - ModelResourceStateDescriptor, ResourceState, SteadyStateException, ValidateModelMixin -) - -# Constants ################################################################### - -PROTOCOL_CHOICES = ( - ('http', 'HTTP - Unencrypted clear text'), - ('https', 'HTTPS - Encrypted'), -) - -gandi = GandiAPI() +from .utils import ValidateModelMixin # Logging ##################################################################### @@ -62,575 +36,93 @@ logger = logging.getLogger(__name__) -# Exceptions ################################################################## - -class InconsistentInstanceState(Exception): - """ - Indicates that the status of an instance can't be determined - """ - pass - +# Models ###################################################################### -# States ###################################################################### -class InstanceState(ResourceState): - """ - A [finite state machine] state describing an instance. +class InstanceReference(TimeStampedModel): """ - # A state is "steady" if we don't expect it to change. - # This information can be used to: - # - delay execution of an operation until the target server reaches a steady state - # - raise an exception when trying to schedule an operation that depends on a state change - # while the target server is in a steady state - # Steady states include: - # - Status.New - # - Status.Running - # - Status.ConfigurationFailed - # - Status.Error - # - Status.Terminated - is_steady_state = True + InstanceReference: Holds common fields and provides a list of all Instances - # An instance is healthy if it is part of a normal (expected) workflow. - # This information can be used to detect problems and highlight them in the UI or notify users. - # Healthy states include: - # - Status.New - # - Status.WaitingForServer - # - Status.ConfiguringServer - # - Status.Running - # - Status.Terminated - is_healthy_state = True + Has name, created, and modified fields for each Instance. - -class Status(ResourceState.Enum): - """ - The states that an instance can be in. + Instance is an abstract class, so having this common InstanceReference class give us a fully + generic way to iterate through all instances and allow instances to be implemented using a + variety of different python classes and database tables. """ + name = models.CharField(max_length=250, blank=False, default='Instance') + instance_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + instance_id = models.PositiveIntegerField() + instance = GenericForeignKey('instance_type', 'instance_id') - class New(InstanceState): - """ Newly created """ - state_id = 'new' - - class WaitingForServer(InstanceState): - """ Server not yet accessible """ - state_id = 'waiting' - name = 'Waiting for server' - is_steady_state = False - - class ConfiguringServer(InstanceState): - """ Running Ansible playbooks on server """ - state_id = 'configuring' - name = 'Configuring server' - is_steady_state = False - - class Running(InstanceState): - """ Instance is up and running """ - state_id = 'running' - - class ConfigurationFailed(InstanceState): - """ Instance was not configured successfully (but may be partially online) """ - state_id = 'failed' - name = 'Configuration failed' - is_healthy_state = False - - class Error(InstanceState): - """ Instance never got up and running (something went wrong when trying to build new VM) """ - state_id = 'error' - is_healthy_state = False - - class Terminated(InstanceState): - """ Instance was running successfully and has been shut down """ - state_id = 'terminated' - - -# Models ###################################################################### + class Meta: + ordering = ['-created'] + unique_together = ('instance_type', 'instance_id') -class Instance(ValidateModelMixin, TimeStampedModel): - """ - Instance - Group of servers running an application made of multiple services +class Instance(ValidateModelMixin): """ - Status = Status - status = ModelResourceStateDescriptor( - state_classes=Status.states, default_state=Status.New, model_field_name='_status' - ) - _status = models.CharField( - max_length=20, - default=status.default_state_class.state_id, - choices=status.model_field_choices, - db_index=True, - db_column='status', - ) - # State transitions: - _status_to_waiting_for_server = status.transition( - from_states=( - Status.New, - Status.Error, - Status.ConfigurationFailed, - Status.Running, - Status.Terminated - ), - to_state=Status.WaitingForServer - ) - _status_to_error = status.transition( - from_states=Status.WaitingForServer, to_state=Status.Error - ) - _status_to_configuring_server = status.transition( - from_states=Status.WaitingForServer, to_state=Status.ConfiguringServer - ) - _status_to_configuration_failed = status.transition( - from_states=Status.ConfiguringServer, to_state=Status.ConfigurationFailed - ) - _status_to_running = status.transition( - from_states=Status.ConfiguringServer, to_state=Status.Running - ) - _status_to_terminated = status.transition( - from_states=Status.Running, to_state=Status.Terminated - ) + Instance: A web application or suite of web applications. - sub_domain = models.CharField(max_length=50) - email = models.EmailField(default='contact@example.com') - name = models.CharField(max_length=250) + An 'Instance' consists of an 'active' AppServer which is available at the instance's URL and + handles all requests from users; the instance may also own some 'terminated' AppServers that + are no longer used, and 'upcoming' AppServers that are used for testing before being + designated as 'active'. - base_domain = models.CharField(max_length=50, blank=True) - protocol = models.CharField(max_length=5, default='http', choices=PROTOCOL_CHOICES) - - last_provisioning_started = models.DateTimeField(blank=True, null=True) + In the future, we may add a scalable instance type, which owns a pool of active AppServers + that all handle requests; currently at most one AppServer is active at any time. + """ + # ref.id should be used instead of id in most places, so rename the default 'id' field. + inst_id = models.IntegerField(primary_key=True) + # Reverse accessor to get the 'InstanceReference' set. This is a 1:1 relation, so use the + # 'ref' property instead of accessing this directly. The only time to use this directly is + # in a query, e.g. to do .select_related('ref_set') + ref_set = GenericRelation(InstanceReference, content_type_field='instance_type', object_id_field='instance_id') class Meta: abstract = True - unique_together = ('base_domain', 'sub_domain') def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = InstanceLoggerAdapter(logger, {'obj': self}) - def __str__(self): - return '{0.name} ({0.url})'.format(self) + @cached_property + def ref(self): + """ Get the InstanceReference for this Instance """ + try: + return self.ref_set.get() # self.ref_set is not aware this is a 1:1 relationship. pylint: disable=no-member + except InstanceReference.DoesNotExist: + return InstanceReference(instance=self) @property - def domain(self): - """ - Instance domain name - """ - return '{0.sub_domain}.{0.base_domain}'.format(self) + def name(self): + """ Get this instance's name, which is stored in the InstanceReference """ + return self.ref.name - @property - def url(self): - """ - Instance URL - """ - return u'{0.protocol}://{0.domain}/'.format(self) + @name.setter + def name(self, new_name): + """ Change the 'name' """ + self.ref.name = new_name @property - def active_server_set(self): - """ - Returns the subset of `self.server_set` which aren't terminated - """ - return self.server_set.exclude_terminated() + def created(self): + """ Get this instance's created date, which is stored in the InstanceReference """ + return self.ref.created @property - def _current_server(self): - """ - Current active server. Raises InconsistentInstanceState if more than - one exists. - """ - active_server_set = self.active_server_set - if not active_server_set: - return - elif active_server_set.count() > 1: - raise InconsistentInstanceState('Multiple servers are active, which is unsupported') - else: - return active_server_set[0] + def modified(self): + """ Get this instance's modified date, which is stored in the InstanceReference """ + return self.ref.modified - @property - def server_status(self): - """ - Instance status - """ - server = self._current_server - if server: - return server.status - return None + def save(self, *args, **kwargs): + """ Save this Instance """ + super().save(*args, **kwargs) + # Ensure an InstanceReference exists, and update its 'modified' field: + self.ref.save() @property def event_context(self): """ Context dictionary to include in events """ - return {'instance_id': self.pk} - - def save(self, **kwargs): - """ - Set default values before saving the instance. - """ - # Set default field values from settings - using the `default` field attribute confuses - # automatically generated migrations, generating a new one when settings don't match - if not self.base_domain: - self.base_domain = settings.INSTANCES_BASE_DOMAIN - super().save(**kwargs) - - def _get_log_entries(self, level_list=None, limit=None): - """ - Return the list of log entry instances for the instance and its current active server, - optionally filtering by logging level. If a limit is given, only the latest records are - returned. - - Returns oldest entries first. - """ - # TODO: Filter out log entries for which the user doesn't have view rights - instance_type = ContentType.objects.get_for_model(self) - server_type = ContentType.objects.get(app_label='instance', model='openstackserver') - server_ids = [server.pk for server in self.server_set.all()] - entries = LogEntry.objects.filter( - (Q(content_type=instance_type) & Q(object_id=self.pk)) | - (Q(content_type=server_type) & Q(object_id__in=server_ids)) - ) - if level_list: - entries = entries.filter(level__in=level_list) - if limit: - return reversed(list(entries[:limit])) - return entries.order_by('created') - - @property - def log_entries(self): - """ - Return the list of log entry instances for the instance and its current active server - """ - return self._get_log_entries(limit=settings.LOG_LIMIT) - - @property - def log_error_entries(self): - """ - Return the list of error or critical log entry instances for the instance and its current - active server - """ - return self._get_log_entries(level_list=['ERROR', 'CRITICAL']) - - -# pylint: disable=too-many-instance-attributes -class SingleVMOpenEdXInstance(MySQLInstanceMixin, MongoDBInstanceMixin, SwiftContainerInstanceMixin, - AnsibleInstanceMixin, GitHubInstanceMixin, EmailInstanceMixin, Instance): - """ - A single instance running a set of Open edX services - """ - forum_version = models.CharField(max_length=50, default='master') - notifier_version = models.CharField(max_length=50, default='master') - xqueue_version = models.CharField(max_length=50, default='master') - certs_version = models.CharField(max_length=50, default='master') - - s3_access_key = models.CharField(max_length=50, blank=True) - s3_secret_access_key = models.CharField(max_length=50, blank=True) - s3_bucket_name = models.CharField(max_length=50, blank=True) - - use_ephemeral_databases = models.BooleanField() - - ANSIBLE_SETTINGS = AnsibleInstanceMixin.ANSIBLE_SETTINGS + [ - 'ansible_s3_settings', - 'ansible_mysql_settings', - 'ansible_mongo_settings', - 'ansible_swift_settings', - ] - - class ProvisionMessages(object): - """ - Class holding ProvisionMessages - """ - PROVISION_EXCEPTION = u"Instance provision failed due to unhandled exception" - PROVISION_ERROR = u"Instance deploy method returned non-zero exit code - provision failed" - - class Meta: - verbose_name = 'Open edX Instance' - ordering = ['-created'] - - @property - def default_fork(self): - """ - Name of the fork to use by default, when no repository is specified - """ - return settings.DEFAULT_FORK - - @property - def ansible_s3_settings(self): - """ - Ansible settings for the S3 bucket - """ - if not self.s3_access_key or not self.s3_secret_access_key or not self.s3_bucket_name: - return '' - - template = loader.get_template('instance/ansible/s3.yml') - return template.render({'instance': self}) - - @property - def ansible_mysql_settings(self): - """ - Ansible settings for the external mysql database - """ - if self.use_ephemeral_databases or not settings.INSTANCE_MYSQL_URL_OBJ: - return '' - - template = loader.get_template('instance/ansible/mysql.yml') - return template.render({'user': self.mysql_user, - 'pass': self.mysql_pass, - 'host': settings.INSTANCE_MYSQL_URL_OBJ.hostname, - 'port': settings.INSTANCE_MYSQL_URL_OBJ.port or 3306, - 'database': self.mysql_database_name}) - - @property - def ansible_mongo_settings(self): - """ - Ansible settings for the external mongo database - """ - if self.use_ephemeral_databases or not settings.INSTANCE_MONGO_URL_OBJ: - return '' - - template = loader.get_template('instance/ansible/mongo.yml') - return template.render({'user': self.mongo_user, - 'pass': self.mongo_pass, - 'host': settings.INSTANCE_MONGO_URL_OBJ.hostname, - 'port': settings.INSTANCE_MONGO_URL_OBJ.port or 27017, - 'database': self.mongo_database_name, - 'forum_database': self.forum_database_name}) - - @property - def ansible_swift_settings(self): - """ - Ansible settings for Swift access. - """ - if self.use_ephemeral_databases or not settings.SWIFT_ENABLE: - return '' - - template = loader.get_template('instance/ansible/swift.yml') - return template.render({'user': self.swift_openstack_user, - 'password': self.swift_openstack_password, - 'tenant': self.swift_openstack_tenant, - 'auth_url': self.swift_openstack_auth_url, - 'region': self.swift_openstack_region}) - - @property - def studio_sub_domain(self): - """ - Studio sub-domain name (eg. 'studio.master') - """ - return 'studio.{}'.format(self.sub_domain) - - @property - def studio_domain(self): - """ - Studio full domain name (eg. 'studio.master.sandbox.opencraft.com') - """ - return '{0.studio_sub_domain}.{0.base_domain}'.format(self) - - @property - def studio_url(self): - """ - Studio URL - """ - return u'{0.protocol}://{0.studio_domain}/'.format(self) - - @property - def database_name(self): - """ - The database name used for external databases. Escape all non-ascii characters and truncate to 64 chars, the - maximum for mysql - """ - name = self.domain.replace('.', '_') - allowed = string.ascii_letters + string.digits + '_' - escaped = ''.join(char for char in name if char in allowed) - return truncate_name(escaped, length=64) - - @property - def mysql_database_name(self): - """ - The mysql database name for this instance - """ - return self.database_name - - @property - def mysql_database_names(self): - """ - List of mysql database names - """ - return [self.mysql_database_name] - - @property - def mongo_database_name(self): - """ - The name of the main external mongo database - """ - return self.database_name - - @property - def forum_database_name(self): - """ - The name of the external database used for forums - """ - return '{0}_forum'.format(self.database_name) - - @property - def mongo_database_names(self): - """ - List of mongo database names - """ - return [self.mongo_database_name, self.forum_database_name] - - @property - def swift_container_name(self): - """ - The name of the Swift container used by the instance. - """ - return self.database_name - - @property - def swift_container_names(self): - """ - The list of Swift container names to be created. - """ - return [self.swift_container_name] - - def save(self, **kwargs): - """ - Set this instance's default field values - """ - if self.use_ephemeral_databases is None: - self.use_ephemeral_databases = settings.INSTANCE_EPHEMERAL_DATABASES - super().save(**kwargs) - - def update_from_pr(self, pr): - """ - Update this instance with settings from the given pull request - """ - super().update_from_pr(pr) - self.ansible_extra_settings = pr.extra_settings - self.use_ephemeral_databases = pr.use_ephemeral_databases(self.domain) - self.ansible_source_repo_url = pr.get_extra_setting('edx_ansible_source_repo') - self.configuration_version = pr.get_extra_setting('configuration_version') - - @log_exception - def provision(self): - """ - Run the provisioning sequence of the instance, recreating the servers from scratch - - Returns: (server, log) - """ - attempt_num = 1 - provisioned = False - server, logs = None, [] - - while attempt_num <= self.attempts and not provisioned: - self.logger.info( - 'Provision attempt {attempt} of {attempts}'.format(attempt=attempt_num, attempts=self.attempts) - ) - server, deploy_log, provisioned = self._provision_attempt() - if deploy_log is not None: # If server fails to build, no deployment logs will be available - logs.extend(deploy_log) - attempt_num += 1 - - return (server, logs) - - def _provision_attempt(self): - """ - Runs single provisioning attempt, creating servers from scratch - - Returns: (server, log) - """ - self.last_provisioning_started = timezone.now() - - # Instance - self.logger.info('Prepare for (re-)provisioning') - self._status_to_waiting_for_server() - - # Server - self.logger.info('Terminate servers') - self.server_set.terminate() - self.logger.info('Start new server') - server = self.server_set.create() - server.start() - - try: - server.sleep_until(lambda: server.status.vm_available) - except SteadyStateException: - self._status_to_error() - return (server, None, False) # No deploy logs available yet; instance has not been provisioned - - def accepts_ssh_commands(): - """ Does server accept SSH commands? """ - return server.status.accepts_ssh_commands - - try: - # DNS - self.logger.info('Waiting for IP assignment on server %s...', server) - server.sleep_until(accepts_ssh_commands) - self.logger.info('Updating DNS: LMS at %s...', self.domain) - gandi.set_dns_record(type='A', name=self.sub_domain, value=server.public_ip) - self.logger.info('Updating DNS: Studio at %s...', self.studio_domain) - gandi.set_dns_record(type='CNAME', name=self.studio_sub_domain, value=self.sub_domain) - - # Provisioning (external databases) - if not self.use_ephemeral_databases: - self.logger.info('Provisioning MySQL database...') - self.provision_mysql() - self.logger.info('Provisioning MongoDB databases...') - self.provision_mongo() - self.logger.info('Provisioning Swift container...') - self.provision_swift() - - # Provisioning (ansible) - self.logger.info('Provisioning server...') - self._status_to_configuring_server() - self.reset_ansible_settings(commit=True) - deploy_log, exit_code = self.deploy() - if exit_code != 0: - self.logger.info('Provisioning failed') - self._status_to_configuration_failed() - self.provision_failed_email(self.ProvisionMessages.PROVISION_ERROR, deploy_log) - return (server, deploy_log, False) - - # Reboot - self.logger.info('Provisioning completed') - self.logger.info('Rebooting server %s...', server) - server.reboot() - server.sleep_until(accepts_ssh_commands) - - # Declare instance up and running - self._status_to_running() - - return (server, deploy_log, True) - - except: - self.server_set.terminate() - self.provision_failed_email(self.ProvisionMessages.PROVISION_EXCEPTION) - raise - - def provision_mysql(self): - """ - Set mysql credentials and provision the database. - """ - if not self.mysql_provisioned: - self.mysql_user = get_random_string(length=16, allowed_chars=string.ascii_lowercase) - self.mysql_pass = get_random_string(length=32) - return super().provision_mysql() - - def provision_mongo(self): - """ - Set mongo credentials and provision the database. - """ - if not self.mongo_provisioned: - self.mongo_user = get_random_string(length=16, allowed_chars=string.ascii_lowercase) - self.mongo_pass = get_random_string(length=32) - return super().provision_mongo() - - def provision_swift(self): - """ - Set Swfit credentials and create the Swift container. - """ - if settings.SWIFT_ENABLE and not self.swift_provisioned: - # TODO: Figure out a way to use separate credentials for each instance. Access control - # on Swift containers is granted to users, and there doesn't seem to be a way to create - # Keystone users in OpenStack public clouds. - self.swift_openstack_user = settings.SWIFT_OPENSTACK_USER - self.swift_openstack_password = settings.SWIFT_OPENSTACK_PASSWORD - self.swift_openstack_tenant = settings.SWIFT_OPENSTACK_TENANT - self.swift_openstack_auth_url = settings.SWIFT_OPENSTACK_AUTH_URL - self.swift_openstack_region = settings.SWIFT_OPENSTACK_REGION - return super().provision_swift() + return {'instance_id': self.ref.pk} diff --git a/instance/models/mixins/ansible.py b/instance/models/mixins/ansible.py index 36ea263c8..cca0e4325 100644 --- a/instance/models/mixins/ansible.py +++ b/instance/models/mixins/ansible.py @@ -20,98 +20,63 @@ Instance app model mixins - Ansible """ import os +from collections import namedtuple from django.conf import settings -from django.core.validators import MinValueValidator from django.db import models -from django.template import loader from instance import ansible from instance.repo import open_repository from instance.utils import poll_streams -class AnsibleInstanceMixin(models.Model): +Playbook = namedtuple('Playbook', [ + 'source_repo', # Path or URL to a git repository contianing the playbook to run + 'playbook_filename', # Relative path to the playbook within source_repo + 'requirements_path', # Relative path to a python requirements file to install before running the playbook + 'version', # the git tag/commit hash/branch to use + 'variables', # a YAML string containing extra variables to pass to ansible when running this playbook +]) + + +class AnsibleAppServerMixin(models.Model): """ An instance that relies on Ansible to deploy its services """ - ansible_source_repo_url = models.URLField(max_length=256, blank=True) - configuration_version = models.CharField(max_length=50, blank=True) - ansible_playbook_name = models.CharField(max_length=50, default='edx_sandbox') - ansible_extra_settings = models.TextField(blank=True) - ansible_settings = models.TextField(blank=True) - - attempts = models.SmallIntegerField(default=3, validators=[ - MinValueValidator(1), - ]) - - # List of attributes to include in the settings output - ANSIBLE_SETTINGS = ['ansible_extra_settings'] - class Meta: abstract = True - def save(self, **kwargs): - """ - Set default values before saving the instance. + def get_playbooks(self): # pylint: disable=no-self-use """ - # Set default field values from settings - using the `default` field attribute confuses - # automatically generated migrations, generating a new one when settings don't match - if not self.ansible_source_repo_url: - self.ansible_source_repo_url = settings.DEFAULT_CONFIGURATION_REPO_URL - if not self.configuration_version: - self.configuration_version = settings.DEFAULT_CONFIGURATION_VERSION - super().save(**kwargs) + Get a list of Playbook objects which describe the playbooks to run in order to install + apps onto this AppServer. - @property - def ansible_playbook_filename(self): + Subclasses should override this like: + return super().get_playbooks + [ + Playbook(source_repo="...", ...) + ] """ - File name of the ansible playbook - """ - return '{}.yml'.format(self.ansible_playbook_name) + return [] @property def inventory_str(self): """ The ansible inventory (list of servers) as a string """ - inventory = ['[app]'] - server_model = self.server_set.model - for server in self.server_set.filter(_status=server_model.Status.Ready.state_id).order_by('created'): - inventory.append(server.public_ip) - inventory_str = '\n'.join(inventory) - self.logger.debug('Inventory:\n%s', inventory_str) - return inventory_str - - def reset_ansible_settings(self, commit=True): - """ - Set the ansible_settings field from the Ansible vars template. - """ - template = loader.get_template('instance/ansible/vars.yml') - vars_str = template.render({ - 'instance': self, - # This proerty is needed twice in the template. To avoid evaluating it twice (and - # querying the Github API twice), we pass it as a context variable. - 'github_admin_username_list': self.github_admin_username_list, - }) - for attr_name in self.ANSIBLE_SETTINGS: - additional_vars = getattr(self, attr_name) - vars_str = ansible.yaml_merge(vars_str, additional_vars) - self.logger.debug('Vars.yml:\n%s', vars_str) - self.ansible_settings = vars_str - if commit: - self.save() + return '[app]\n{server_ip}'.format(server_ip=self.server.public_ip) - def _run_playbook(self, requirements_path, playbook_path): + def _run_playbook(self, working_dir, playbook): """ Run a playbook against the instance active servers """ + playbook_path = os.path.join(working_dir, playbook.playbook_filename) + log_lines = [] with ansible.run_playbook( - requirements_path, - self.inventory_str, - self.ansible_settings, - playbook_path, - self.ansible_playbook_filename, + requirements_path=os.path.join(working_dir, playbook.requirements_path), + inventory_str=self.inventory_str, + vars_str=playbook.variables, + playbook_path=os.path.dirname(playbook_path), + playbook_name=os.path.basename(playbook_path), username=settings.OPENSTACK_SANDBOX_SSH_USERNAME, ) as process: try: @@ -134,19 +99,20 @@ def _run_playbook(self, requirements_path, playbook_path): process.wait() return log_lines, process.returncode - def deploy(self): + def run_ansible_playbooks(self): """ - Deploy instance to the active servers + Provision the server using ansible """ - with open_repository(self.ansible_source_repo_url, ref=self.configuration_version) as configuration_repo: - playbook_path = os.path.join(configuration_repo.working_dir, 'playbooks') - requirements_path = os.path.join(configuration_repo.working_dir, 'requirements.txt') - - self.logger.info( - 'Running playbook "{path}/{name}":'.format(path=playbook_path, name=self.ansible_playbook_name) - ) + log = "" + for playbook in self.get_playbooks(): + with open_repository(playbook.source_repo, ref=playbook.version) as configuration_repo: + self.logger.info('Running playbook "%s" from "%s"', playbook.playbook_filename, playbook.source_repo) + new_log, returncode = self._run_playbook(configuration_repo.working_dir, playbook) + log += new_log + if returncode != 0: + self.logger.error('Playbook failed for AppServer %s', self) + break - log, returncode = self._run_playbook(requirements_path, playbook_path) - playbook_result = 'completed' if returncode == 0 else 'failed' - self.logger.error('Playbook {result} for instance {instance}'.format(result=playbook_result, instance=self)) + if returncode == 0: + self.logger.info('Playbooks completed for AppServer %s', self) return (log, returncode) diff --git a/instance/models/mixins/openedx_database.py b/instance/models/mixins/openedx_database.py new file mode 100644 index 000000000..a1b3f484e --- /dev/null +++ b/instance/models/mixins/openedx_database.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +# +# OpenCraft -- tools to aid developing and hosting free software projects +# Copyright (C) 2015 OpenCraft +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +""" +Open edX instance database mixin +""" +import string + +from django.conf import settings +from django.db.backends.utils import truncate_name +from django.template import loader +from django.utils.crypto import get_random_string + +from .database import MySQLInstanceMixin, MongoDBInstanceMixin, SwiftContainerInstanceMixin + + +# pylint: disable=too-many-instance-attributes +class OpenEdXDatabasesMixin(MySQLInstanceMixin, MongoDBInstanceMixin, SwiftContainerInstanceMixin): + """ + Mixin that provides functionality required for all the databases that an OpenEdX Instance + uses (when not using ephemeral databases) + """ + + @property + def database_name(self): + """ + The database name used for external databases. + """ + name = self.domain.replace('.', '_') + # Escape all non-ascii characters and truncate to 64 chars, the maximum for mysql: + allowed = string.ascii_letters + string.digits + '_' + escaped = ''.join(char for char in name if char in allowed) + return truncate_name(escaped, length=64) + + @property + def mysql_database_name(self): + """ + The mysql database name for this instance + """ + return self.database_name + + @property + def mysql_database_names(self): + """ + List of mysql database names + """ + return [self.mysql_database_name] + + @property + def mongo_database_name(self): + """ + The name of the main external mongo database + """ + return self.database_name + + @property + def forum_database_name(self): + """ + The name of the external database used for forums + """ + return '{0}_forum'.format(self.database_name) + + @property + def mongo_database_names(self): + """ + List of mongo database names + """ + return [self.mongo_database_name, self.forum_database_name] + + @property + def swift_container_name(self): + """ + The name of the Swift container used by the instance. + """ + return self.database_name + + @property + def swift_container_names(self): + """ + The list of Swift container names to be created. + """ + return [self.swift_container_name] + + def provision_mysql(self): + """ + Set mysql credentials and provision the database. + """ + if not self.mysql_provisioned: + self.mysql_user = get_random_string(length=16, allowed_chars=string.ascii_lowercase) + self.mysql_pass = get_random_string(length=32) + return super().provision_mysql() + + def provision_mongo(self): + """ + Set mongo credentials and provision the database. + """ + if not self.mongo_provisioned: + self.mongo_user = get_random_string(length=16, allowed_chars=string.ascii_lowercase) + self.mongo_pass = get_random_string(length=32) + return super().provision_mongo() + + def provision_swift(self): + """ + Set Swift credentials and create the Swift container. + """ + if settings.SWIFT_ENABLE and not self.swift_provisioned: + # TODO: Figure out a way to use separate credentials for each instance. Access control + # on Swift containers is granted to users, and there doesn't seem to be a way to create + # Keystone users in OpenStack public clouds. + self.swift_openstack_user = settings.SWIFT_OPENSTACK_USER + self.swift_openstack_password = settings.SWIFT_OPENSTACK_PASSWORD + self.swift_openstack_tenant = settings.SWIFT_OPENSTACK_TENANT + self.swift_openstack_auth_url = settings.SWIFT_OPENSTACK_AUTH_URL + self.swift_openstack_region = settings.SWIFT_OPENSTACK_REGION + return super().provision_swift() + + def get_database_settings(self): + """ + Get configuration_database_settings to pass to a new AppServer + + Only needed when not using ephemeral databases + """ + if self.use_ephemeral_databases: + return '' + + new_settings = '' + + # MySQL: + if settings.INSTANCE_MYSQL_URL_OBJ: + template = loader.get_template('instance/ansible/mysql.yml') + new_settings += template.render({ + 'user': self.mysql_user, + 'pass': self.mysql_pass, + 'host': settings.INSTANCE_MYSQL_URL_OBJ.hostname, + 'port': settings.INSTANCE_MYSQL_URL_OBJ.port or 3306, + 'database': self.mysql_database_name + }) + + # MongoDB: + if settings.INSTANCE_MONGO_URL_OBJ: + template = loader.get_template('instance/ansible/mongo.yml') + new_settings += template.render({ + 'user': self.mongo_user, + 'pass': self.mongo_pass, + 'host': settings.INSTANCE_MONGO_URL_OBJ.hostname, + 'port': settings.INSTANCE_MONGO_URL_OBJ.port or 27017, + 'database': self.mongo_database_name, + 'forum_database': self.forum_database_name + }) + + # S3: + if self.s3_access_key and self.s3_secret_access_key and self.s3_bucket_name: + template = loader.get_template('instance/ansible/s3.yml') + new_settings += template.render({'instance': self}) + + # SWIFT: + if settings.SWIFT_ENABLE: + template = loader.get_template('instance/ansible/swift.yml') + return template.render({ + 'user': self.swift_openstack_user, + 'password': self.swift_openstack_password, + 'tenant': self.swift_openstack_tenant, + 'auth_url': self.swift_openstack_auth_url, + 'region': self.swift_openstack_region + }) + + return new_settings diff --git a/instance/models/mixins/utilities.py b/instance/models/mixins/utilities.py index 99b5a3408..60e326ffd 100644 --- a/instance/models/mixins/utilities.py +++ b/instance/models/mixins/utilities.py @@ -26,22 +26,22 @@ from django.views.debug import ExceptionReporter -class EmailInstanceMixin(object): +class EmailMixin: """ - An instance class that can send emails + An AppServer class that can send emails """ class EmailSubject(object): """ Class holding email subject constants """ - PROVISION_FAILED = u"Instance {instance_name} ({instance_url}) failed to provision" + PROVISION_FAILED = "AppServer {name} ({instance_name}) failed to provision" class EmailBody(object): """ Class holding email body constants """ - PROVISION_FAILED = u"Instance {instance_name} failed to provision.\n" \ - u"Reason: {reason}\n" + PROVISION_FAILED = "Instance {instance_name} AppServer {name} failed to provision.\n" \ + "Reason: {reason}\n" @staticmethod def _get_exc_info(default=None): @@ -65,8 +65,8 @@ def provision_failed_email(self, reason, log=None): attachments.append(("provision.log", log_str, "text/plain")) self._send_email( - self.EmailSubject.PROVISION_FAILED.format(instance_name=self.name, instance_url=self.url), - self.EmailBody.PROVISION_FAILED.format(instance_name=self.name, reason=reason), + self.EmailSubject.PROVISION_FAILED.format(name=self.name, instance_name=self.instance.name), + self.EmailBody.PROVISION_FAILED.format(name=self.name, instance_name=self.instance.name, reason=reason), self._get_exc_info(default=None), attachments=attachments ) diff --git a/instance/models/openedx_appserver.py b/instance/models/openedx_appserver.py new file mode 100644 index 000000000..3965cdf63 --- /dev/null +++ b/instance/models/openedx_appserver.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- +# +# OpenCraft -- tools to aid developing and hosting free software projects +# Copyright (C) 2015 OpenCraft +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +""" +Instance app models - Open EdX AppServer models +""" +from django.conf import settings +from django.db import models +from django.template import loader +from django.utils.text import slugify + +from instance import ansible +from instance.logging import log_exception +from .appserver import AppServer +from .mixins.ansible import AnsibleAppServerMixin, Playbook +from .mixins.utilities import EmailMixin +from .utils import SteadyStateException + +# Constants ################################################################### + +PROTOCOL_CHOICES = ( + ('http', 'HTTP - Unencrypted clear text'), + ('https', 'HTTPS - Encrypted'), +) + +# Models ###################################################################### + + +class OpenEdXAppConfiguration(models.Model): + """ + Configuration fields used by OpenEdX Instance and AppServer. + + Mutable on the instance but immutable on the AppServer. + """ + class Meta: + abstract = True + + email = models.EmailField(default='contact@example.com', help_text=( + 'The default contact email for this instance; also used as the from address for emails ' + 'sent by the server.' + )) + protocol = models.CharField(max_length=5, default='http', choices=PROTOCOL_CHOICES) + + # Ansible-specific settings: + configuration_source_repo_url = models.URLField(max_length=256, blank=False) + configuration_version = models.CharField(max_length=50, blank=False) + configuration_extra_settings = models.TextField(blank=True, help_text="YAML config vars that override all others") + + edx_platform_repository_url = models.CharField(max_length=256, blank=False, help_text=( + 'URL to the edx-platform repository to use. Leave blank for default.' + )) + edx_platform_commit = models.CharField(max_length=256, blank=False, help_text=( + 'edx-platform commit hash or branch or tag to use. Leave blank to use the default, ' + 'which is equal to the value of "openedx_release".' + )) + + openedx_release = models.CharField(max_length=128, blank=False, help_text=(""" + Set this to a release tag like "named-release/dogwood" to build a specific release of + Open edX. The default is "{default}". This setting becomes the default value for + edx_platform_version, forum_version, notifier_version, xqueue_version, and + certs_version so it should be a git branch that exists in all of + those repositories. + + Note: to build a specific branch of edx-platform, you should just override + edx_platform_commit rather than changing this setting. + + Note 2: This value does not affect the default value of configuration_version. + """).format(default=settings.DEFAULT_OPENEDX_RELEASE)) + + s3_access_key = models.CharField(max_length=50, blank=True) + s3_secret_access_key = models.CharField(max_length=50, blank=True) + s3_bucket_name = models.CharField(max_length=50, blank=True) + + # Misc settings: + use_ephemeral_databases = models.BooleanField() + + def set_field_defaults(self): + """ + Set default values. + """ + if not self.openedx_release: + self.openedx_release = settings.DEFAULT_OPENEDX_RELEASE + if not self.configuration_source_repo_url: + self.configuration_source_repo_url = settings.DEFAULT_CONFIGURATION_REPO_URL + if not self.configuration_version: + self.configuration_version = settings.DEFAULT_CONFIGURATION_VERSION + if not self.edx_platform_repository_url: + self.edx_platform_repository_url = settings.DEFAULT_EDX_PLATFORM_REPO_URL + if not self.edx_platform_commit: + self.edx_platform_commit = self.openedx_release + super().set_field_defaults() + + +class OpenEdXAppServer(AppServer, OpenEdXAppConfiguration, AnsibleAppServerMixin, EmailMixin): + """ + OpenEdXAppServer: One or more of the Open edX apps, running on a single VM + + Typically, most of the Open edX apps are enabled, including but not limited to: + * edxapp (LMS+Studio) + * cs_comments_service (forums) + * notifier + * xqueue + * insights + """ + name = models.CharField(max_length=250) + + configuration_database_settings = models.TextField(blank=True, help_text="YAML vars for file and DB configuration") + # configuration_settings: A record of the combined (final) ansible variables passed to + # the configuration playbook when configuring this AppServer. + configuration_settings = models.TextField(blank=False) + + CONFIGURATION_PLAYBOOK = 'edx_sandbox' + CONFIGURATION_VARS_TEMPLATE = 'instance/ansible/vars.yml' + # Additional model fields/properties that contain yaml vars to add the the configuration vars: + CONFIGURATION_EXTRA_FIELDS = [ + 'configuration_database_settings', + 'configuration_extra_settings', + ] + + def set_field_defaults(self): + """ + Set default values. + """ + # Always override configuration_settings - it's not meant to be manually set. We can't + # assert that it isn't set because if a ValidationError occurred, this method could be + # called multiple times before this AppServer is successfully created. + self.configuration_settings = self.create_configuration_settings() + super().set_field_defaults() + + def get_playbooks(self): + """ + Get the ansible playbooks used to provision this AppServer + """ + return super().get_playbooks() + [ + Playbook( + source_repo=self.configuration_source_repo_url, + requirements_path='requirements.txt', + playbook_filename='playbooks/{}.yml'.format(self.CONFIGURATION_PLAYBOOK), + version=self.configuration_version, + variables=self.configuration_settings, + ) + ] + + def create_configuration_settings(self): + """ + Generate the configuration settings. + + This is a one-time thing, because configuration_settings, like all AppServer fields, is + immutable once this AppServer is saved. + """ + template = loader.get_template(self.CONFIGURATION_VARS_TEMPLATE) + vars_str = template.render({ + 'instance': self, # TODO: Rename this template var to 'appserver' + # This property is needed twice in the template. To avoid evaluating it twice (and + # querying the Github API twice), we pass it as a context variable. + 'github_admin_username_list': self.instance.github_admin_username_list, + }) + for attr_name in self.CONFIGURATION_EXTRA_FIELDS: + additional_vars = getattr(self, attr_name) + vars_str = ansible.yaml_merge(vars_str, additional_vars) + self.logger.debug('Vars.yml:\n%s', vars_str) + return vars_str + + @log_exception + @AppServer.status.only_for(AppServer.Status.New) + def provision(self): + """ + Provision this AppServer. + + Returns True on success or False on failure + """ + self.logger.info('Starting provisioning') + # Start by requesting a new server/VM: + self._status_to_waiting_for_server() + assert self.server_id is None + server_prefix = ('edxapp-' + slugify(self.instance.sub_domain))[:20] + self.server.create(name_prefix=server_prefix) + self.server.start() + + def accepts_ssh_commands(): + """ Does server accept SSH commands? """ + return self.server.status.accepts_ssh_commands + + try: + self.logger.info('Waiting for server %s...', self.server) + self.server.sleep_until(lambda: self.server.status.vm_available) + self.logger.info('Waiting for server %s to finish booting...', self.server) + self.server.sleep_until(accepts_ssh_commands) + except SteadyStateException: + self._status_to_error() + return False + + try: + # Provisioning (ansible) + self.logger.info('Provisioning server...') + self._status_to_configuring_server() + log, exit_code = self.run_ansible_playbooks() + if exit_code != 0: + self.logger.info('Provisioning failed') + self._status_to_configuration_failed() + self.provision_failed_email("AppServer deploy failed: ansible play exited with non-zero exit code", log) + return False + + # Reboot + self.logger.info('Provisioning completed') + self.logger.info('Rebooting server %s...', self.server) + self.server.reboot() + self.server.sleep_until(accepts_ssh_commands) + + # Declare instance up and running + self._status_to_running() + + return True + + except: + self.server.terminate() + self.provision_failed_email("AppServer deploy failed: unhandled exception") + raise diff --git a/instance/models/openedx_instance.py b/instance/models/openedx_instance.py new file mode 100644 index 000000000..bf67d1f4f --- /dev/null +++ b/instance/models/openedx_instance.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# +# OpenCraft -- tools to aid developing and hosting free software projects +# Copyright (C) 2015 OpenCraft +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +""" +Open edX Instance models +""" +from django.conf import settings +from django.db import models + +from instance.gandi import GandiAPI +from instance.logging import log_exception +from .mixins.openedx_database import OpenEdXDatabasesMixin +from .mixins.version_control import GitHubInstanceMixin +from .instance import Instance +from .openedx_appserver import OpenEdXAppConfiguration, OpenEdXAppServer + +# Constants ################################################################### + +gandi = GandiAPI() + +# Models ###################################################################### + + +class BaseOpenEdXInstance(Instance, OpenEdXAppConfiguration, OpenEdXDatabasesMixin): + """ + BaseOpenEdXInstance: represents a website or set of affiliated websites powered by the same + OpenEdX installation. + """ + + # Most settings/fields are inherited from OpenEdXAppConfiguration + sub_domain = models.CharField(max_length=50) + base_domain = models.CharField(max_length=50, blank=True) + + active_appserver = models.OneToOneField(OpenEdXAppServer, null=True, on_delete=models.SET_NULL, related_name='+') + + class Meta: + abstract = True + unique_together = ('base_domain', 'sub_domain') + verbose_name = 'Open edX Instance' + + @property + def domain(self): + """ + Instance domain name + """ + return '{0.sub_domain}.{0.base_domain}'.format(self) + + @property + def url(self): + """ + LMS URL + """ + return u'{0.protocol}://{0.domain}/'.format(self) + + @property + def studio_sub_domain(self): + """ + Studio sub-domain name (eg. 'studio.master') + """ + return 'studio.{}'.format(self.sub_domain) + + @property + def studio_domain(self): + """ + Studio full domain name (eg. 'studio.master.sandbox.opencraft.com') + """ + return '{0.studio_sub_domain}.{0.base_domain}'.format(self) + + @property + def studio_url(self): + """ + Studio URL + """ + return u'{0.protocol}://{0.studio_domain}/'.format(self) + + def save(self, **kwargs): + """ + Set default values before saving the instance. + """ + # Set default field values from settings - using the `default` field attribute confuses + # automatically generated migrations, generating a new one when settings don't match + if not self.base_domain: + self.base_domain = settings.INSTANCES_BASE_DOMAIN + if self.use_ephemeral_databases is None: + self.use_ephemeral_databases = settings.INSTANCE_EPHEMERAL_DATABASES + super().save(**kwargs) + + @property + def appserver_set(self): + """ + Get the set of OpenEdxAppServers owned by this instance. + """ + return self.ref.openedxappserver_set + + def set_appserver_active(self, appserver_id): + """ + Mark the AppServer with the given ID as the active one. + """ + app_server = self.appserver_set.get(pk=appserver_id) # Make sure the AppServer is owned by this instance + self.active_appserver = app_server + self.save() + self.logger.info('Updating DNS: LMS at %s...', self.domain) + gandi.set_dns_record(type='A', name=self.sub_domain, value=self.active_appserver.server.public_ip) + self.logger.info('Updating DNS: Studio at %s...', self.studio_domain) + gandi.set_dns_record(type='CNAME', name=self.studio_sub_domain, value=self.sub_domain) + + @log_exception + def spawn_appserver(self): + """ + Provision a new AppServer. If it completes successfully, mark it as active. + """ + # Provision external databases: + if not self.use_ephemeral_databases: + # TODO: Use db row-level locking to ensure we don't get any race conditions when creating these DBs. + # Use select_for_update(nowait=True) to lock this object's row, then do these steps, then refresh_from_db + self.logger.info('Provisioning MySQL database...') + self.provision_mysql() + self.logger.info('Provisioning MongoDB databases...') + self.provision_mongo() + self.logger.info('Provisioning Swift container...') + self.provision_swift() + + config_fields = [field.name for field in OpenEdXAppConfiguration._meta.fields if field.name not in ('id', )] + config_copy = {field_name: getattr(self, field_name) for field_name in config_fields} + + app_server = self.appserver_set.create( + # Name for the app server: this should usually generate a unique name (and won't cause any issues if not): + name="AppServer {}".format(self.appserver_set.count() + 1), + # Copy the current value of each setting into the AppServer, preserving it permanently: + configuration_database_settings=self.get_database_settings(), + **config_copy + ) + if app_server.provision(): + # If the AppServer provisioned successfully, make it the active one: + # Note: if I call spawn_appserver() twice, and the second one provisions sooner, the first one may then + # finish and replace the second as the active server. We are not really worried about that for now. + self.set_appserver_active(app_server.pk) + + +class SandboxInstance(BaseOpenEdXInstance, GitHubInstanceMixin): + """ + SandboxInstance: represents an Open edX instance created for a pull request. + """ + + class Meta: + verbose_name = 'Open edX Sandbox Instance' + + def update_from_pr(self, pr): + """ + Update this instance with settings from the given pull request + """ + super().update_from_pr(pr) + self.configuration_extra_settings = pr.extra_settings + self.use_ephemeral_databases = pr.use_ephemeral_databases(self.domain) + self.configuration_source_repo_url = pr.get_extra_setting('edx_ansible_source_repo') + self.configuration_version = pr.get_extra_setting('configuration_version') + + @property + def default_fork(self): + """ + Name of the fork to use by default, when no repository is specified + """ + return settings.DEFAULT_FORK diff --git a/instance/models/server.py b/instance/models/server.py index 92079f96c..ac9201ffb 100644 --- a/instance/models/server.py +++ b/instance/models/server.py @@ -38,7 +38,6 @@ from instance.logger_adapter import ServerLoggerAdapter from instance.utils import is_port_open, to_json -from instance.models.instance import SingleVMOpenEdXInstance from instance.models.utils import ( ValidateModelMixin, ResourceState, ModelResourceStateDescriptor, SteadyStateException ) @@ -142,6 +141,8 @@ class Server(ValidateModelMixin, TimeStampedModel): """ A single server VM """ + name_prefix = models.SlugField(max_length=20, blank=False) + Status = Status status = ModelResourceStateDescriptor( state_classes=Status.states, default_state=Status.Pending, model_field_name='_status' @@ -163,8 +164,6 @@ class Server(ValidateModelMixin, TimeStampedModel): from_states=(Status.Building, Status.Booting, Status.Ready), to_state=Status.Unknown ) - instance = models.ForeignKey(SingleVMOpenEdXInstance, related_name='server_set') - objects = ServerQuerySet().as_manager() logger = ServerLoggerAdapter(logger, {}) @@ -177,15 +176,18 @@ def __init__(self, *args, **kwargs): self.logger = ServerLoggerAdapter(logger, {'obj': self}) + @property + def name(self): + """ Get a name for this server (slug-friendly) """ + assert self.id is not None + return "{prefix}-{num}".format(prefix=self.name_prefix, num=self.id) + @property def event_context(self): """ Context dictionary to include in events """ - return { - 'instance_id': self.instance.pk, - 'server_id': self.pk, - } + return {'server_id': self.pk} def sleep_until(self, condition, timeout=3600): """ @@ -327,7 +329,7 @@ def start(self): try: os_server = openstack.create_server( self.nova, - self.instance.sub_domain, + self.name, settings.OPENSTACK_SANDBOX_FLAVOR, settings.OPENSTACK_SANDBOX_BASE_IMAGE, key_name=settings.OPENSTACK_SANDBOX_SSH_KEYNAME, diff --git a/instance/tasks.py b/instance/tasks.py index 6b9351f3b..12ef5e3cb 100644 --- a/instance/tasks.py +++ b/instance/tasks.py @@ -27,7 +27,7 @@ from django.conf import settings from instance.github import get_username_list_from_team, get_pr_list_from_username -from instance.models.instance import SingleVMOpenEdXInstance +from instance.models.openedx_instance import SandboxInstance # Logging ##################################################################### @@ -44,10 +44,10 @@ def provision_instance(instance_pk): Run provisioning on an existing instance """ logger.info('Retreiving instance: pk=%s', instance_pk) - instance = SingleVMOpenEdXInstance.objects.get(pk=instance_pk) + instance = SandboxInstance.objects.get(pk=instance_pk) - logger.info('Running provisioning on %s', instance) - instance.provision() + logger.info('Spawning new AppServer on %s', instance) + instance.spawn_appserver() @db_periodic_task(crontab(minute='*/1')) @@ -61,7 +61,7 @@ def watch_pr(): for username in team_username_list: for pr in get_pr_list_from_username(username, settings.WATCH_FORK): sub_domain = 'pr{number}.sandbox'.format(number=pr.number) - instance, created = SingleVMOpenEdXInstance.objects.update_or_create_from_pr(pr, sub_domain) + instance, created = SandboxInstance.objects.update_or_create_from_pr(pr, sub_domain) if created: logger.info('New PR found, creating sandbox: %s', pr) provision_instance(instance.pk) diff --git a/instance/templates/instance/ansible/vars.yml b/instance/templates/instance/ansible/vars.yml index 942a8a1e3..e424d77dd 100644 --- a/instance/templates/instance/ansible/vars.yml +++ b/instance/templates/instance/ansible/vars.yml @@ -44,16 +44,16 @@ SECURITY_UPDATE_ALL_PACKAGES: false SECURITY_UPGRADE_ON_ANSIBLE: true # Repositories URLs -edx_ansible_source_repo: '{{ instance.ansible_source_repo_url }}' -edx_platform_repo: '{{ instance.repository_url }}' +edx_ansible_source_repo: '{{ instance.configuration_source_repo_url }}' +edx_platform_repo: '{{ instance.edx_platform_repository_url }}' # Pin down dependencies to specific (known to be compatible) commits. -edx_platform_version: '{{ instance.commit_id }}' +edx_platform_version: '{{ instance.edx_platform_commit }}' configuration_version: '{{ instance.configuration_version }}' -forum_version: '{{ instance.forum_version }}' -notifier_version: '{{ instance.notifier_version }}' -xqueue_version: '{{ instance.xqueue_version }}' -certs_version: '{{ instance.certs_version }}' +forum_version: '{{ instance.openedx_release }}' +notifier_version: '{{ instance.openedx_release }}' +xqueue_version: '{{ instance.openedx_release }}' +certs_version: '{{ instance.openedx_release }}' # Features EDXAPP_FEATURES: diff --git a/opencraft/settings.py b/opencraft/settings.py index c397b49d4..98e24d5cf 100644 --- a/opencraft/settings.py +++ b/opencraft/settings.py @@ -282,7 +282,14 @@ DEFAULT_CONFIGURATION_REPO_URL = env( 'DEFAULT_CONFIGURATION_REPO_URL', default='https://github.com/edx/configuration.git' ) -DEFAULT_CONFIGURATION_VERSION = env('DEFAULT_CONFIGURATION_VERSION', default='master') + +DEFAULT_EDX_PLATFORM_REPO_URL = env( + 'DEFAULT_EDX_PLATFORM_REPO_URL', default='https://github.com/edx/edx-platform.git' +) + +DEFAULT_OPENEDX_RELEASE = env('DEFAULT_OPENEDX_RELEASE', default='master') + +DEFAULT_CONFIGURATION_VERSION = env('DEFAULT_CONFIGURATION_VERSION', default=DEFAULT_OPENEDX_RELEASE) # Ansible #####################################################################