feat: Expect-Staple and Expect-CT support

requirements-base.txt

@@ -47,6 +47,7 @@ simplejson>=3.2.0,<3.9.0
 six>=1.10.0,<1.11.0
 setproctitle>=1.1.7,<1.2.0
 statsd>=3.1.0,<3.2.0
+strict-rfc3339>=0.7
 structlog==16.1.0
 sqlparse>=0.1.16,<0.2.0
 symbolic>=2.0.2,<3.0.0
@@ -58,3 +59,4 @@ rb>=1.7.0,<2.0.0
 qrcode>=5.2.2,<6.0.0
 python-u2flib-server>=4.0.1,<4.1.0
 redis-py-cluster>=1.3.4,<1.4.0
+jsonschema==2.6.0

src/sentry/conf/server.py

@@ -814,7 +814,9 @@ def create_partitioned_queues(name):
 SENTRY_SMTP_PORT = 1025
 
 SENTRY_INTERFACES = {
-    'csp': 'sentry.interfaces.csp.Csp',
+    'csp': 'sentry.interfaces.security.Csp',
+    'expectct': 'sentry.interfaces.security.ExpectCT',
+    'expectstaple': 'sentry.interfaces.security.ExpectStaple',
     'device': 'sentry.interfaces.device.Device',
     'exception': 'sentry.interfaces.exception.Exception',
     'logentry': 'sentry.interfaces.message.Message',
@@ -837,7 +839,7 @@ def create_partitioned_queues(name):
     'sentry.interfaces.Query': 'sentry.interfaces.query.Query',
     'sentry.interfaces.Http': 'sentry.interfaces.http.Http',
     'sentry.interfaces.User': 'sentry.interfaces.user.User',
-    'sentry.interfaces.Csp': 'sentry.interfaces.csp.Csp',
+    'sentry.interfaces.Csp': 'sentry.interfaces.security.Csp',
     'sentry.interfaces.AppleCrashReport': 'sentry.interfaces.applecrash.AppleCrashReport',
     'sentry.interfaces.Breadcrumbs': 'sentry.interfaces.breadcrumbs.Breadcrumbs',
     'sentry.interfaces.Contexts': 'sentry.interfaces.contexts.Contexts',

src/sentry/coreapi.py

@@ -11,10 +11,11 @@
 from __future__ import absolute_import, print_function
 
 import base64
+import jsonschema
 import logging
+import re
 import six
 import zlib
-import re
 
 from collections import MutableMapping
 from django.core.exceptions import SuspiciousOperation
@@ -25,14 +26,13 @@
 
 from sentry import filters
 from sentry.cache import default_cache
-from sentry.interfaces.csp import Csp
+from sentry.interfaces.base import get_interface
 from sentry.event_manager import EventManager
 from sentry.models import ProjectKey
 from sentry.tasks.store import preprocess_event, \
     preprocess_event_from_reprocessing
 from sentry.utils import json
 from sentry.utils.auth import parse_auth_header
-from sentry.utils.csp import is_valid_csp_report
 from sentry.utils.http import origin_from_request
 from sentry.utils.data_filters import is_valid_ip, \
     is_valid_release, is_valid_error_message, FilterStatKeys
@@ -382,9 +382,14 @@ def auth_from_request(self, request):
         return auth
 
 
-class CspApiHelper(ClientApiHelper):
+class SecurityApiHelper(ClientApiHelper):
+
+    report_interfaces = ('sentry.interfaces.Csp', 'expectct', 'expectstaple')
+
     def origin_from_request(self, request):
-        # We don't use an origin here
+        # In the case of security reports, the origin is not available at the
+        # dispatch() stage, as we need to parse it out of the request body, so
+        # we do our own CORS check once we have parsed it.
         return None
 
     def auth_from_request(self, request):
@@ -401,48 +406,56 @@ def auth_from_request(self, request):
         return auth
 
     def should_filter(self, project, data, ip_address=None):
-        if not is_valid_csp_report(data['sentry.interfaces.Csp'], project):
-            return (True, FilterStatKeys.INVALID_CSP)
-        return super(CspApiHelper, self).should_filter(project, data, ip_address)
+        for name in self.report_interfaces:
+            if name in data:
+                interface = get_interface(name)
+                if interface.to_python(data[name]).should_filter(project):
+                    return (True, FilterStatKeys.INVALID_CSP)
 
-    def validate_data(self, data):
-        # pop off our meta data used to hold Sentry specific stuff
-        meta = data.pop('_meta', {})
+        return super(SecurityApiHelper, self).should_filter(project, data, ip_address)
 
-        # All keys are sent with hyphens, so we want to conver to underscores
-        report = {k.replace('-', '_'): v for k, v in six.iteritems(data)}
+    def validate_data(self, data):
+        try:
+            interface = get_interface(data.pop('interface'))
+            report = data.pop('report')
+        except KeyError:
+            raise APIForbidden('No report or interface data')
 
+        # To support testing, we can either accept a buillt interface instance, or the raw data in
+        # which case we build the instance ourselves
         try:
-            inst = Csp.to_python(report)
-        except Exception as exc:
-            raise APIForbidden('Invalid CSP Report: %s' % exc)
-
-        # Construct a faux Http interface based on the little information we have
-        headers = {}
-        if self.context.agent:
-            headers['User-Agent'] = self.context.agent
-        if inst.referrer:
-            headers['Referer'] = inst.referrer
-
-        data = {
+            instance = report if isinstance(report, interface) else interface.from_raw(report)
+        except jsonschema.ValidationError as e:
+            raise APIError('Invalid security report: %s' % str(e).splitlines()[0])
+
+        def clean(d):
+            return dict(filter(lambda x: x[1], d.items()))
+
+        data.update({
             'logger': 'csp',
-            'message': inst.get_message(),
-            'culprit': inst.get_culprit(),
-            'release': meta.get('release'),
-            'tags': inst.get_tags(),
-            inst.get_path(): inst.to_json(),
+            'message': instance.get_message(),
+            'culprit': instance.get_culprit(),
+            instance.get_path(): instance.to_json(),
+            'tags': instance.get_tags(),
+            'errors': [],
+
+            'sentry.interfaces.User': {
+                'ip_address': self.context.ip_address,
+            },
+
+            # Construct a faux Http interface based on the little information we have
             # This is a bit weird, since we don't have nearly enough
             # information to create an Http interface, but
             # this automatically will pick up tags for the User-Agent
             # which is actually important here for CSP
             'sentry.interfaces.Http': {
-                'url': inst.document_uri,
-                'headers': headers,
+                'url': instance.get_origin(),
+                'headers': clean({
+                    'User-Agent': self.context.agent,
+                    'Referer': instance.get_referrer(),
+                })
             },
-            'sentry.interfaces.User': {
-                'ip_address': self.context.ip_address,
-            },
-        }
+        })
 
         return data
 

src/sentry/eventtypes/__init__.py

@@ -1,13 +1,15 @@
 from __future__ import absolute_import
 
 from .base import DefaultEvent
-from .csp import CspEvent
+from .security import CspEvent, ExpectCTEvent, ExpectStapleEvent
 from .error import ErrorEvent
 from .manager import EventTypeManager
 
 # types are ordered by priority, default should always be last
 default_manager = EventTypeManager()
 default_manager.register(CspEvent)
+default_manager.register(ExpectCTEvent)
+default_manager.register(ExpectStapleEvent)
 default_manager.register(ErrorEvent)
 default_manager.register(DefaultEvent)
 

src/sentry/eventtypes/security.py

@@ -0,0 +1,62 @@
+from __future__ import absolute_import
+
+from .base import BaseEvent
+
+
+class CspEvent(BaseEvent):
+    key = 'csp'
+
+    def has_metadata(self):
+        # TODO(alexh) also look for 'csp' ?
+        return 'sentry.interfaces.Csp' in self.data
+
+    def get_metadata(self):
+        from sentry.interfaces.security import Csp
+        # TODO(dcramer): pull get message into here to avoid instantiation
+        # or ensure that these get interfaces passed instead of raw data
+        csp = Csp.to_python(self.data['sentry.interfaces.Csp'])
+
+        return {
+            'directive': csp.effective_directive,
+            'uri': csp._normalized_blocked_uri,
+            'message': csp.get_message(),
+        }
+
+    def to_string(self, metadata):
+        return metadata['message']
+
+
+class ExpectCTEvent(BaseEvent):
+    key = 'expectct'
+
+    def has_metadata(self):
+        return 'expectct' in self.data
+
+    def get_metadata(self):
+        from sentry.interfaces.security import ExpectCT
+        expectct = ExpectCT.to_python(self.data['expectct'])
+        return {
+            'origin': expectct.get_origin(),
+            'message': expectct.get_message(),
+        }
+
+    def to_string(self, metadata):
+        return metadata['message']
+
+
+class ExpectStapleEvent(BaseEvent):
+    key = 'expectstaple'
+
+    def has_metadata(self):
+        return 'expectstaple' in self.data
+
+    def get_metadata(self):
+        from sentry.interfaces.security import ExpectStaple
+        expectstaple = ExpectStaple.to_python(self.data['expectstaple'])
+        return {
+            'origin': expectstaple.get_origin(),
+            'message': expectstaple.get_message(),
+        }
+
+    def to_string(self, metadata):
+        return metadata['message']

src/sentry/interfaces/base.py

@@ -11,17 +11,6 @@
 from sentry.utils.safe import safe_execute
 
 
-def iter_interfaces():
-    rv = {}
-
-    for name, import_path in six.iteritems(settings.SENTRY_INTERFACES):
-        rv.setdefault(import_path, []).append(name)
-
-    for import_path, keys in six.iteritems(rv):
-        iface = import_string(import_path)
-        yield iface, keys
-
-
 def get_interface(name):
     try:
         import_path = settings.SENTRY_INTERFACES[name]
@@ -99,7 +88,7 @@ def __setattr__(self, name, value):
 
     @classmethod
     def to_python(cls, data):
-        return cls(data)
+        return cls(**data)
 
     def get_api_context(self, is_public=False):
         return self.to_json()