Add start of work to add webhook modeling for each integration

media/css/core.css

@@ -119,6 +119,12 @@ h3 > span.link-help {
     float: right;
 }
 
+form.form-wide input[type='text'],
+form.form-wide select,
+form.form-wide textarea {
+    width: 100%;
+}
+
 /* content */
 
 #content { padding-top: 50px; }
@@ -925,6 +931,22 @@ body .edit-toggle { display: none; }
 
 .navigable ul input[type=text] { width: 164px; }
 
+div.button-bar ul {
+    list-style: none;
+    text-align: right;
+}
+
+div.button-bar ul li {
+    display: inline-block;
+}
+
+div.button-bar li a.button,
+div.button-bar li input[type="submit"],
+div.button-bar li input[type="button"],
+div.button-bar li button {
+    margin-top: .5em;
+    margin-bottom: .5em;
+}
 
 select.dropdown { display: none; }
 .dropdown > a { font-family: "ff-meta-web-pro", "ff-meta-web-pro-1", "ff-meta-web-pro-2", Arial, "Helvetica Neue", sans-serif; color: #666; font-weight: bold; padding: 8px 15px; border: none; background: #e6e6e6 url(../images/gradient.png) repeat-x bottom left; margin: 30px 5px 20px 0; text-shadow: 0 1px 0 rgba(255, 255, 255, 1); border: 1px solid #bfbfbf; display: block; text-decoration: none; box-shadow: 0 1px 0 rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.5) inset; -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.5) inset; -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.5) inset; }

readthedocs/constants.py

@@ -9,4 +9,5 @@
     'lang_slug': LANGUAGES_REGEX,
     'version_slug': VERSION_SLUG_REGEX,
     'filename_slug': '(?:.*)',
+    'integer_pk': r'[\d]+',
 }

readthedocs/integrations/admin.py

@@ -0,0 +1,103 @@
+"""Integration admin models"""
+
+from datetime import datetime, timedelta
+
+from django.contrib import admin
+from django.contrib.contenttypes.models import ContentType
+from django.core import urlresolvers
+from django.utils.safestring import mark_safe
+from django.utils.translation import ugettext_lazy as _
+from pygments.formatters import HtmlFormatter
+
+from .models import Integration, HttpExchange
+
+
+def pretty_json_field(field, description, include_styles=False):
+    # There is some styling here because this is easier than reworking how the
+    # admin is getting stylesheets. We only need minimal styles here, and there
+    # isn't much user impact to these styles as well.
+    def inner(self, obj):
+        styles = ''
+        if include_styles:
+            formatter = HtmlFormatter(style='colorful')
+            styles = '<style>' + formatter.get_style_defs() + '</style>'
+        return mark_safe('<div style="{0}">{1}</div>{2}'.format(
+            'float: left;',
+            obj.formatted_json(field),
+            styles,
+        ))
+
+    inner.short_description = description
+    return inner
+
+
+class HttpExchangeAdmin(admin.ModelAdmin):
+
+    readonly_fields = [
+        'date',
+        'status_code',
+        'pretty_request_headers',
+        'pretty_request_body',
+        'pretty_response_headers',
+        'pretty_response_body',
+    ]
+    fields = readonly_fields
+    list_display = [
+        'related_object',
+        'date',
+        'status_code',
+        'failed_icon',
+    ]
+
+    pretty_request_headers = pretty_json_field(
+        'request_headers',
+        'Request headers',
+        include_styles=True,
+    )
+    pretty_request_body = pretty_json_field(
+        'request_body',
+        'Request body',
+    )
+    pretty_response_headers = pretty_json_field(
+        'response_headers',
+        'Response headers',
+    )
+    pretty_response_body = pretty_json_field(
+        'response_body',
+        'Response body',
+    )
+
+    def failed_icon(self, obj):
+        return not obj.failed
+
+    failed_icon.boolean = True
+    failed_icon.short_description = 'Passed'
+
+
+class IntegrationAdmin(admin.ModelAdmin):
+
+    search_fields = ('project__slug', 'project__name')
+    readonly_fields = ['exchanges']
+
+    def exchanges(self, obj):
+        """Manually make an inline-ish block
+
+        JSONField doesn't do well with fieldsets for whatever reason. This is
+        just to link to the exchanges.
+        """
+        url = urlresolvers.reverse('admin:{0}_{1}_changelist'.format(
+            HttpExchange._meta.app_label,
+            HttpExchange._meta.model_name,
+        ))
+        return mark_safe('<a href="{0}?{1}={2}">{3} HTTP transactions</a>'.format(
+            url,
+            'integrations',
+            obj.pk,
+            obj.exchanges.count(),
+        ))
+
+    exchanges.short_description = 'HTTP exchanges'
+
+
+admin.site.register(HttpExchange, HttpExchangeAdmin)
+admin.site.register(Integration, IntegrationAdmin)

readthedocs/integrations/migrations/0002_add-webhook.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.12 on 2017-03-29 21:29
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('integrations', '0001_add_http_exchange'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Integration',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('integration_type', models.CharField(choices=[(b'github_webhook', 'GitHub incoming webhook'), (b'bitbucket_webhook', 'Bitbucket incoming webhook'), (b'gitlab_webhook', 'GitLab incoming webhook'), (b'api_webhook', 'Generic API incoming webhook')], max_length=32, verbose_name='Type')),
+                ('provider_data', jsonfield.fields.JSONField(verbose_name='Provider data')),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='integrations', to='projects.Project')),
+            ],
+        ),
+    ]

readthedocs/integrations/models.py

@@ -2,10 +2,11 @@
 
 import json
 import uuid
+import re
 
 from django.db import models, transaction
 from django.utils.translation import ugettext_lazy as _
-from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
 from django.contrib.contenttypes.models import ContentType
 from django.utils.safestring import mark_safe
 from rest_framework import status
@@ -14,13 +15,20 @@
 from pygments.lexers import JsonLexer
 from pygments.formatters import HtmlFormatter
 
+from readthedocs.projects.models import Project
 from .utils import normalize_request_payload
 
 
 class HttpExchangeManager(models.Manager):
 
     """HTTP exchange manager methods"""
 
+    # Filter rules for request headers to remove from the output
+    REQ_FILTER_RULES = [
+        re.compile('^X-Forwarded-.*$', re.I),
+        re.compile('^X-Real-Ip$', re.I),
+    ]
+
     @transaction.atomic
     def from_exchange(self, req, resp, related_object, payload=None):
         """Create object from Django request and response objects
@@ -54,6 +62,11 @@ def from_exchange(self, req, resp, related_object, payload=None):
             if key.startswith('HTTP_')
         )
         request_headers['Content-Type'] = req.content_type
+        # Remove unwanted headers
+        for filter_rule in self.REQ_FILTER_RULES:
+            for key in request_headers.keys():
+                if filter_rule.match(key):
+                    del request_headers[key]
 
         response_payload = resp.data if hasattr(resp, 'data') else resp.content
         try:
@@ -75,14 +88,16 @@ def from_exchange(self, req, resp, related_object, payload=None):
         return obj
 
     def delete_limit(self, related_object, limit=10):
-        # pylint: disable=protected-access
-        queryset = self.filter(
-            content_type=ContentType.objects.get(
-                app_label=related_object._meta.app_label,
-                model=related_object._meta.model_name,
-            ),
-            object_id=related_object.pk
-        )
+        if isinstance(related_object, Integration):
+            queryset = self.filter(integrations=related_object)
+        else:
+            queryset = self.filter(
+                content_type=ContentType.objects.get(
+                    app_label=related_object._meta.app_label,
+                    model=related_object._meta.model_name,
+                ),
+                object_id=related_object.pk
+            )
         for exchange in queryset[limit:]:
             exchange.delete()
 
@@ -126,7 +141,9 @@ def formatted_json(self, field):
         """Try to return pretty printed and Pygment highlighted code"""
         value = getattr(self, field) or ''
         try:
-            json_value = json.dumps(json.loads(value), sort_keys=True, indent=2)
+            if not isinstance(value, dict):
+                value = json.loads(value)
+            json_value = json.dumps(value, sort_keys=True, indent=2)
             formatter = HtmlFormatter()
             html = highlight(json_value, JsonLexer(), formatter)
             return mark_safe(html)
@@ -140,3 +157,106 @@ def formatted_request_body(self):
     @property
     def formatted_response_body(self):
         return self.formatted_json('response_body')
+
+
+class IntegrationQuerySet(models.QuerySet):
+
+    """Return a subclass of Integration, based on the integration type
+
+    .. note::
+        This doesn't affect queries currently, only fetching of an object
+    """
+
+    def get(self, *args, **kwargs):
+        """Replace model instance on Integration subclasses
+
+        This is based on the ``integration_type`` field, and is used to provide
+        specific functionality to and integration via a proxy subclass of the
+        Integration model.
+        """
+        old = super(IntegrationQuerySet, self).get(*args, **kwargs)
+        # Build a mapping of integration_type -> class dynamically
+        class_map = dict(
+            (cls.integration_type_id, cls)
+            for cls in self.model.__subclasses__()
+            if hasattr(cls, 'integration_type_id')
+        )
+        cls_replace = class_map.get(old.integration_type)
+        if cls_replace is None:
+            return old
+        new = cls_replace()
+        for k, v in old.__dict__.items():
+            new.__dict__[k] = v
+        return new
+
+
+class Integration(models.Model):
+
+    """Inbound webhook integration for projects"""
+
+    GITHUB_WEBHOOK = 'github_webhook'
+    BITBUCKET_WEBHOOK = 'bitbucket_webhook'
+    GITLAB_WEBHOOK = 'gitlab_webhook'
+    API_WEBHOOK = 'api_webhook'
+
+    WEBHOOK_INTEGRATIONS = (
+        (GITHUB_WEBHOOK, _('GitHub incoming webhook')),
+        (BITBUCKET_WEBHOOK, _('Bitbucket incoming webhook')),
+        (GITLAB_WEBHOOK, _('GitLab incoming webhook')),
+        (API_WEBHOOK, _('Generic API incoming webhook')),
+    )
+
+    INTEGRATIONS = WEBHOOK_INTEGRATIONS
+
+    project = models.ForeignKey(Project, related_name='integrations')
+    integration_type = models.CharField(
+        _('Integration type'),
+        max_length=32,
+        choices=INTEGRATIONS
+    )
+    provider_data = JSONField(_('Provider data'))
+    exchanges = GenericRelation(
+        'HttpExchange',
+        related_query_name='integrations'
+    )
+
+    objects = IntegrationQuerySet.as_manager()
+
+    # Integration attributes
+    has_sync = False
+
+    def __unicode__(self):
+        return (_('{0} for {1}')
+                .format(self.get_integration_type_display(), self.project.name))
+
+
+class GitHubWebhook(Integration):
+
+    integration_type_id = Integration.GITHUB_WEBHOOK
+    has_sync = True
+
+    class Meta:
+        proxy = True
+
+    @property
+    def can_sync(self):
+        try:
+            return all((k in self.provider_data) for k in ['id', 'url'])
+        except (ValueError, TypeError):
+            return False
+
+
+class BitbucketWebhook(Integration):
+
+    integration_type_id = Integration.BITBUCKET_WEBHOOK
+    has_sync = True
+
+    class Meta:
+        proxy = True
+
+    @property
+    def can_sync(self):
+        try:
+            return all((k in self.provider_data) for k in ['id', 'url'])
+        except (ValueError, TypeError):
+            return False

readthedocs/oauth/services/base.py

@@ -125,6 +125,9 @@ def create_organization(self, fields):
     def setup_webhook(self, project):
         raise NotImplementedError
 
+    def update_webhook(self, project, integration):
+        raise NotImplementedError
+
     @classmethod
     def is_project_service(cls, project):
         """Determine if this is the service the project is using