Cleanups

instance/logger_adapter.py

@@ -24,9 +24,32 @@
 
 import logging
 
+# Helper methods ##############################################################
+
+
+def format_instance(instance):
+    """ Given any concrete subclass of Instance, get a short ID string to put into the log """
+    if instance:
+        return 'instance={} ({!s:.15})'.format(instance.ref.pk, instance.ref.name)
+    return 'Unknown Instance'
+
+
+def format_appserver(app_server):
+    """ Given an AppServer subclass, get a short ID string to put into the log """
+    if app_server:
+        return 'app_server={} ({!s:.15})'.format(app_server.pk, app_server.name)
+    return 'Unknown AppServer'
+
+
+def format_server(server):
+    """ Given a Server subclass, get a short ID string to put into the log """
+    if server:
+        return 'server={!s:.20}'.format(server.name)
+    return 'Unknown Server'
 
 # Adapters ####################################################################
 
+
 class AppServerLoggerAdapter(logging.LoggerAdapter):
     """
     Custom LoggerAdapter for Instance objects
@@ -41,6 +64,10 @@ def process(self, msg, kwargs):
         else:
             return msg, kwargs
 
+        app_server = self.extra['obj']
+        msg = '{},{} | {}'.format(format_instance(app_server.instance), format_appserver(app_server), msg)
+        return msg, kwargs
+
 
 class ServerLoggerAdapter(logging.LoggerAdapter):
     """
@@ -51,7 +78,18 @@ def process(self, msg, kwargs):
         msg, kwargs = super().process(msg, kwargs)
 
         server = self.extra['obj']
-        if server.instance and server.instance.sub_domain:
-            return 'instance={!s:.15},server={!s:.8} | {}'.format(server.instance.sub_domain, server, msg), kwargs
-        else:
-            return msg, kwargs
+        msg = '{} | {}'.format(format_server(server), msg)
+        return msg, kwargs
+
+
+class InstanceLoggerAdapter(logging.LoggerAdapter):
+    """
+    Custom LoggerAdapter for Instance objects
+    Include the InstanceReference ID in the output
+    """
+    def process(self, msg, kwargs):
+        msg, kwargs = super().process(msg, kwargs)
+
+        instance = self.extra['obj']
+        msg = '{} | {}'.format(format_instance(instance), msg)
+        return msg, kwargs

instance/models/appserver.py

@@ -23,24 +23,18 @@
 # Imports #####################################################################
 
 import logging
-import string
 
 from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
 from django.db import models
-from django.db.backends.utils import truncate_name
-from django.template import loader
-from django.utils import timezone
-from django.utils.crypto import get_random_string
+from django.db.models import Q
 from django_extensions.db.models import TimeStampedModel
-from functools import partial
 
 from instance.logger_adapter import AppServerLoggerAdapter
-from instance.logging import log_exception
+from .instance import InstanceReference
 from .log_entry import LogEntry
-from .mixins.utilities import EmailInstanceMixin
-from .mixins.version_control import GitHubInstanceMixin
 from .server import OpenStackServer
-from .utils import ModelResourceStateDescriptor, ResourceState, SteadyStateException, ValidateModelMixin
+from .utils import ModelResourceStateDescriptor, ResourceState, ValidateModelMixin
 
 
 # Logging #####################################################################
@@ -164,10 +158,10 @@ class AppServer(ValidateModelMixin, TimeStampedModel):
         from_states=Status.Running, to_state=Status.Terminated
     )
 
-    name = models.CharField(max_length=250)
-    server = models.OneToOneField(OpenStackServer, on_delete=models.CASCADE, related_name='owner')
+    name = models.CharField(max_length=250, blank=False)
+    server = models.OneToOneField(OpenStackServer, on_delete=models.CASCADE, related_name='+')
     # The Instance that owns this. Instance will get related_name accessors like 'openedxappserver_set'
-    owner = models.ForeignKey(Instance, on_delete=models.CASCADE, related_name='%(class)s_set')
+    owner = models.ForeignKey(InstanceReference, on_delete=models.CASCADE, related_name='%(class)s_set')
 
     class Meta:
         abstract = True
@@ -185,14 +179,16 @@ def instance(self):
         """
         Get the Instance that owns this AppServer
         """
-        return self.owner
+        return self.owner.instance
 
     @property
     def event_context(self):
         """
         Context dictionary to include in events
         """
-        return {'appserver_id': self.pk, 'instance_id': self.instance.pk}
+        context = self.instance.event_context  # dict with instance_id
+        context.update({'appserver_id': self.pk, 'appserver_type': self.__class__.__name__})
+        return context
 
     def set_field_defaults(self):
         """
@@ -228,9 +224,11 @@ def _get_log_entries(self, level_list=None, limit=None):
         returned.
         """
         # TODO: Filter out log entries for which the user doesn't have view rights
+        appserver_type = ContentType.objects.get_for_model(self)
+        server_type = ContentType.objects.get_for_model(self.server)
         entries = LogEntry.objects.filter(
-            (Q(category=LogEntry.CATEGORY_APPSERVER) & Q(obj_id=self.pk)) |
-            (Q(category=LogEntry.CATEGORY_SERVER) & Q(obj_id=self.server_id))
+            (Q(content_type=appserver_type) & Q(object_id=self.pk)) |
+            (Q(content_type=server_type) & Q(object_id=self.server_id))
         )
         if level_list:
             entries = entries.filter(level__in=level_list)
@@ -252,10 +250,3 @@ def log_error_entries(self):
         server it manages
         """
         return self._get_log_entries(level_list=['ERROR', 'CRITICAL'])
-
-    # class ProvisionMessages(object):
-    #     """
-    #     Class holding ProvisionMessages
-    #     """
-    #     PROVISION_EXCEPTION = u"Instance provision failed due to unhandled exception"
-    #     PROVISION_ERROR = u"Instance deploy method returned non-zero exit code - provision failed"

instance/models/instance.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+#
+# OpenCraft -- tools to aid developing and hosting free software projects
+# Copyright (C) 2015 OpenCraft <xavier@opencraft.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+Instance app models - Open EdX Instance and AppServer models
+"""
+import logging
+
+from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
+from django.contrib.contenttypes.models import ContentType
+from django.db import models
+from django.utils.functional import cached_property
+from django_extensions.db.models import TimeStampedModel
+
+from instance.logger_adapter import InstanceLoggerAdapter
+from .utils import ValidateModelMixin
+
+
+# Logging #####################################################################
+
+logger = logging.getLogger(__name__)
+
+
+# Models ######################################################################
+
+
+class InstanceReference(TimeStampedModel):
+    """
+    InstanceReference: Holds common fields and provides a list of all Instances
+
+    Has name, created, and modified fields for each Instance.
+
+    Instance is an abstract class, so having this common InstanceReference class give us a fully
+    generic way to iterate through all instances and allow instances to be implemented using a
+    variety of different python classes and database tables.
+    """
+    name = models.CharField(max_length=250, blank=False, default='Instance')
+    instance_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
+    instance_id = models.PositiveIntegerField()
+    instance = GenericForeignKey('instance_type', 'instance_id')
+
+    class Meta:
+        ordering = ['-created']
+        unique_together = ('instance_type', 'instance_id')
+
+
+class Instance(ValidateModelMixin):
+    """
+    Instance: A web application or suite of web applications.
+
+    An 'Instance' consists of an 'active' AppServer which is available at the instance's URL and
+    handles all requests from users; the instance may also own some 'terminated' AppServers that
+    are no longer used, and 'upcoming' AppServers that are used for testing before being
+    designated as 'active'.
+
+    In the future, we may add a scalable instance type, which owns a pool of active AppServers
+    that all handle requests; currently at most one AppServer is active at any time.
+    """
+    # ref.id should be used instead of id in most places, so rename the default 'id' field.
+    inst_id = models.IntegerField(primary_key=True)
+    # Reverse accessor to get the 'InstanceReference' set. This is a 1:1 relation, so use the
+    # 'ref' property instead of accessing this directly. The only time to use this directly is
+    # in a query, e.g. to do .select_related('ref_set')
+    ref_set = GenericRelation(InstanceReference, content_type_field='instance_type', object_id_field='instance_id')
+
+    class Meta:
+        abstract = True
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.logger = InstanceLoggerAdapter(logger, {'obj': self})
+
+    @cached_property
+    def ref(self):
+        """ Get the InstanceReference for this Instance """
+        try:
+            return self.ref_set.get()  # self.ref_set is not aware this is a 1:1 relationship. pylint: disable=no-member
+        except InstanceReference.DoesNotExist:
+            return InstanceReference(instance=self)
+
+    @property
+    def name(self):
+        """ Get this instance's name, which is stored in the InstanceReference """
+        return self.ref.name
+
+    @name.setter
+    def name(self, new_name):
+        """ Change the 'name' """
+        self.ref.name = new_name
+
+    @property
+    def created(self):
+        """ Get this instance's created date, which is stored in the InstanceReference """
+        return self.ref.created
+
+    @property
+    def modified(self):
+        """ Get this instance's modified date, which is stored in the InstanceReference """
+        return self.ref.modified
+
+    def save(self, *args, **kwargs):
+        """ Save this Instance """
+        super().save(*args, **kwargs)
+        # Ensure an InstanceReference exists, and update its 'modified' field:
+        self.ref.save()
+
+    @property
+    def event_context(self):
+        """
+        Context dictionary to include in events
+        """
+        return {'instance_id': self.ref.pk}

instance/models/log_entry.py

@@ -21,6 +21,7 @@
 """
 
 # Imports #####################################################################
+from django.contrib.contenttypes.models import ContentType
 from django.db import models
 from django_extensions.db.models import TimeStampedModel
 
@@ -41,18 +42,9 @@ class LogEntry(ValidateModelMixin, TimeStampedModel):
         ('CRITICAL', 'Critical'),
     )
 
-    CATEGORY_GENERAL = None
-    CATEGORY_APPSERVER = 1
-    CATEGORY_SERVER = 2
-    CATEGORY_CHOICES = (
-        (CATEGORY_GENERAL, 'General'),  # Single log entry that isn't attached to a specific model, such as instances or servers
-        (CATEGORY_APPSERVER, 'AppServer'),  # log entry owned by an AppServer (subclass)
-        (CATEGORY_SERVER, 'Server'),  # log entry owned by a Server (subclass)
-    )
-
     text = models.TextField(blank=True)
-    category = models.IntegerField(null=True, choices=CATEGORY_CHOICES)
-    obj_id = models.IntegerField(null=True, blank=True, default=None)  # ID of an AppServer or Server or None
+    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, null=True)
+    object_id = models.PositiveIntegerField(null=True)
     level = models.CharField(max_length=9, db_index=True, default='INFO', choices=LOG_LEVEL_CHOICES)
 
     class Meta:

instance/models/mixins/ansible.py

@@ -99,7 +99,7 @@ def _run_playbook(self, working_dir, playbook):
             process.wait()
             return log_lines, process.returncode
 
-    def deploy(self):
+    def run_ansible_playbooks(self):
         """
         Provision the server using ansible
         """