diff --git a/requirements-base.txt b/requirements-base.txt index 41eee59a7995c5..40f9a07ee5ea79 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -47,6 +47,7 @@ simplejson>=3.2.0,<3.9.0 six>=1.10.0,<1.11.0 setproctitle>=1.1.7,<1.2.0 statsd>=3.1.0,<3.2.0 +strict-rfc3339>=0.7 structlog==16.1.0 sqlparse>=0.1.16,<0.2.0 symbolic>=2.0.2,<3.0.0 @@ -58,3 +59,4 @@ rb>=1.7.0,<2.0.0 qrcode>=5.2.2,<6.0.0 python-u2flib-server>=4.0.1,<4.1.0 redis-py-cluster>=1.3.4,<1.4.0 +jsonschema==2.6.0 diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index f416da31899aee..1e7982e3bd78a1 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -814,7 +814,9 @@ def create_partitioned_queues(name): SENTRY_SMTP_PORT = 1025 SENTRY_INTERFACES = { - 'csp': 'sentry.interfaces.csp.Csp', + 'csp': 'sentry.interfaces.security.Csp', + 'expectct': 'sentry.interfaces.security.ExpectCT', + 'expectstaple': 'sentry.interfaces.security.ExpectStaple', 'device': 'sentry.interfaces.device.Device', 'exception': 'sentry.interfaces.exception.Exception', 'logentry': 'sentry.interfaces.message.Message', @@ -837,7 +839,7 @@ def create_partitioned_queues(name): 'sentry.interfaces.Query': 'sentry.interfaces.query.Query', 'sentry.interfaces.Http': 'sentry.interfaces.http.Http', 'sentry.interfaces.User': 'sentry.interfaces.user.User', - 'sentry.interfaces.Csp': 'sentry.interfaces.csp.Csp', + 'sentry.interfaces.Csp': 'sentry.interfaces.security.Csp', 'sentry.interfaces.AppleCrashReport': 'sentry.interfaces.applecrash.AppleCrashReport', 'sentry.interfaces.Breadcrumbs': 'sentry.interfaces.breadcrumbs.Breadcrumbs', 'sentry.interfaces.Contexts': 'sentry.interfaces.contexts.Contexts', diff --git a/src/sentry/coreapi.py b/src/sentry/coreapi.py index 38eb59e81c36b2..18f006dc832c1f 100644 --- a/src/sentry/coreapi.py +++ b/src/sentry/coreapi.py @@ -11,10 +11,11 @@ from __future__ import absolute_import, print_function import base64 +import jsonschema import logging +import re import six import zlib -import re from collections import MutableMapping from django.core.exceptions import SuspiciousOperation @@ -25,14 +26,13 @@ from sentry import filters from sentry.cache import default_cache -from sentry.interfaces.csp import Csp +from sentry.interfaces.base import get_interface from sentry.event_manager import EventManager from sentry.models import ProjectKey from sentry.tasks.store import preprocess_event, \ preprocess_event_from_reprocessing from sentry.utils import json from sentry.utils.auth import parse_auth_header -from sentry.utils.csp import is_valid_csp_report from sentry.utils.http import origin_from_request from sentry.utils.data_filters import is_valid_ip, \ is_valid_release, is_valid_error_message, FilterStatKeys @@ -382,9 +382,14 @@ def auth_from_request(self, request): return auth -class CspApiHelper(ClientApiHelper): +class SecurityApiHelper(ClientApiHelper): + + report_interfaces = ('sentry.interfaces.Csp', 'expectct', 'expectstaple') + def origin_from_request(self, request): - # We don't use an origin here + # In the case of security reports, the origin is not available at the + # dispatch() stage, as we need to parse it out of the request body, so + # we do our own CORS check once we have parsed it. return None def auth_from_request(self, request): @@ -401,48 +406,56 @@ def auth_from_request(self, request): return auth def should_filter(self, project, data, ip_address=None): - if not is_valid_csp_report(data['sentry.interfaces.Csp'], project): - return (True, FilterStatKeys.INVALID_CSP) - return super(CspApiHelper, self).should_filter(project, data, ip_address) + for name in self.report_interfaces: + if name in data: + interface = get_interface(name) + if interface.to_python(data[name]).should_filter(project): + return (True, FilterStatKeys.INVALID_CSP) - def validate_data(self, data): - # pop off our meta data used to hold Sentry specific stuff - meta = data.pop('_meta', {}) + return super(SecurityApiHelper, self).should_filter(project, data, ip_address) - # All keys are sent with hyphens, so we want to conver to underscores - report = {k.replace('-', '_'): v for k, v in six.iteritems(data)} + def validate_data(self, data): + try: + interface = get_interface(data.pop('interface')) + report = data.pop('report') + except KeyError: + raise APIForbidden('No report or interface data') + # To support testing, we can either accept a buillt interface instance, or the raw data in + # which case we build the instance ourselves try: - inst = Csp.to_python(report) - except Exception as exc: - raise APIForbidden('Invalid CSP Report: %s' % exc) - - # Construct a faux Http interface based on the little information we have - headers = {} - if self.context.agent: - headers['User-Agent'] = self.context.agent - if inst.referrer: - headers['Referer'] = inst.referrer - - data = { + instance = report if isinstance(report, interface) else interface.from_raw(report) + except jsonschema.ValidationError as e: + raise APIError('Invalid security report: %s' % str(e).splitlines()[0]) + + def clean(d): + return dict(filter(lambda x: x[1], d.items())) + + data.update({ 'logger': 'csp', - 'message': inst.get_message(), - 'culprit': inst.get_culprit(), - 'release': meta.get('release'), - 'tags': inst.get_tags(), - inst.get_path(): inst.to_json(), + 'message': instance.get_message(), + 'culprit': instance.get_culprit(), + instance.get_path(): instance.to_json(), + 'tags': instance.get_tags(), + 'errors': [], + + 'sentry.interfaces.User': { + 'ip_address': self.context.ip_address, + }, + + # Construct a faux Http interface based on the little information we have # This is a bit weird, since we don't have nearly enough # information to create an Http interface, but # this automatically will pick up tags for the User-Agent # which is actually important here for CSP 'sentry.interfaces.Http': { - 'url': inst.document_uri, - 'headers': headers, + 'url': instance.get_origin(), + 'headers': clean({ + 'User-Agent': self.context.agent, + 'Referer': instance.get_referrer(), + }) }, - 'sentry.interfaces.User': { - 'ip_address': self.context.ip_address, - }, - } + }) return data diff --git a/src/sentry/eventtypes/__init__.py b/src/sentry/eventtypes/__init__.py index bb56f3672603ff..47b3c9d59fe82d 100644 --- a/src/sentry/eventtypes/__init__.py +++ b/src/sentry/eventtypes/__init__.py @@ -1,13 +1,15 @@ from __future__ import absolute_import from .base import DefaultEvent -from .csp import CspEvent +from .security import CspEvent, ExpectCTEvent, ExpectStapleEvent from .error import ErrorEvent from .manager import EventTypeManager # types are ordered by priority, default should always be last default_manager = EventTypeManager() default_manager.register(CspEvent) +default_manager.register(ExpectCTEvent) +default_manager.register(ExpectStapleEvent) default_manager.register(ErrorEvent) default_manager.register(DefaultEvent) diff --git a/src/sentry/eventtypes/csp.py b/src/sentry/eventtypes/csp.py deleted file mode 100644 index 7f5ce2bb36b069..00000000000000 --- a/src/sentry/eventtypes/csp.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import absolute_import - -from .base import BaseEvent - - -class CspEvent(BaseEvent): - key = 'csp' - - def has_metadata(self): - return 'sentry.interfaces.Csp' in self.data - - def get_metadata(self): - # TODO(dcramer): we need to avoid importing interfaces in this module - # due to recursion at top level - from sentry.interfaces.csp import Csp - # TODO(dcramer): pull get message into here to avoid instantiation - # or ensure that these get interfaces passed instead of raw data - csp = Csp.to_python(self.data['sentry.interfaces.Csp']) - - return { - 'directive': csp.effective_directive, - 'uri': csp._normalized_blocked_uri, - 'message': csp.get_message(), - } - - def to_string(self, metadata): - return metadata['message'] diff --git a/src/sentry/eventtypes/security.py b/src/sentry/eventtypes/security.py new file mode 100644 index 00000000000000..8225039659f90e --- /dev/null +++ b/src/sentry/eventtypes/security.py @@ -0,0 +1,62 @@ +from __future__ import absolute_import + +from .base import BaseEvent + + +class CspEvent(BaseEvent): + key = 'csp' + + def has_metadata(self): + # TODO(alexh) also look for 'csp' ? + return 'sentry.interfaces.Csp' in self.data + + def get_metadata(self): + from sentry.interfaces.security import Csp + # TODO(dcramer): pull get message into here to avoid instantiation + # or ensure that these get interfaces passed instead of raw data + csp = Csp.to_python(self.data['sentry.interfaces.Csp']) + + return { + 'directive': csp.effective_directive, + 'uri': csp._normalized_blocked_uri, + 'message': csp.get_message(), + } + + def to_string(self, metadata): + return metadata['message'] + + +class ExpectCTEvent(BaseEvent): + key = 'expectct' + + def has_metadata(self): + return 'expectct' in self.data + + def get_metadata(self): + from sentry.interfaces.security import ExpectCT + expectct = ExpectCT.to_python(self.data['expectct']) + return { + 'origin': expectct.get_origin(), + 'message': expectct.get_message(), + } + + def to_string(self, metadata): + return metadata['message'] + + +class ExpectStapleEvent(BaseEvent): + key = 'expectstaple' + + def has_metadata(self): + return 'expectstaple' in self.data + + def get_metadata(self): + from sentry.interfaces.security import ExpectStaple + expectstaple = ExpectStaple.to_python(self.data['expectstaple']) + return { + 'origin': expectstaple.get_origin(), + 'message': expectstaple.get_message(), + } + + def to_string(self, metadata): + return metadata['message'] diff --git a/src/sentry/interfaces/base.py b/src/sentry/interfaces/base.py index b56962dcf2b92d..413937cf2dabad 100644 --- a/src/sentry/interfaces/base.py +++ b/src/sentry/interfaces/base.py @@ -11,17 +11,6 @@ from sentry.utils.safe import safe_execute -def iter_interfaces(): - rv = {} - - for name, import_path in six.iteritems(settings.SENTRY_INTERFACES): - rv.setdefault(import_path, []).append(name) - - for import_path, keys in six.iteritems(rv): - iface = import_string(import_path) - yield iface, keys - - def get_interface(name): try: import_path = settings.SENTRY_INTERFACES[name] @@ -99,7 +88,7 @@ def __setattr__(self, name, value): @classmethod def to_python(cls, data): - return cls(data) + return cls(**data) def get_api_context(self, is_public=False): return self.to_json() diff --git a/src/sentry/interfaces/csp.py b/src/sentry/interfaces/csp.py deleted file mode 100644 index fb109aa1bc7272..00000000000000 --- a/src/sentry/interfaces/csp.py +++ /dev/null @@ -1,232 +0,0 @@ -""" -sentry.interfaces.csp -~~~~~~~~~~~~~~~~~~~~~ - -:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details. -:license: BSD, see LICENSE for more details. -""" - -from __future__ import absolute_import - -__all__ = ('Csp', ) - -from six.moves.urllib.parse import urlsplit, urlunsplit - -from sentry.interfaces.base import Interface -from sentry.utils import json -from sentry.utils.cache import memoize -from sentry.utils.safe import trim -from sentry.web.helpers import render_to_string - -# Sourced from https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives -REPORT_KEYS = frozenset( - ( - 'blocked_uri', - 'document_uri', - 'effective_directive', - 'original_policy', - 'referrer', - 'status_code', - 'violated_directive', - 'source_file', - 'line_number', - 'column_number', - - # FireFox specific keys - 'script_sample', - ) -) - -KEYWORDS = frozenset(("'none'", "'self'", "'unsafe-inline'", "'unsafe-eval'", )) - -ALL_SCHEMES = ('data:', 'mediastream:', 'blob:', 'filesystem:', 'http:', 'https:', 'file:', ) - -SELF = "'self'" - -DIRECTIVE_TO_MESSAGES = { - # 'base-uri': '', - 'child-src': (u"Blocked 'child' from '{uri}'", "Blocked inline 'child'"), - 'connect-src': (u"Blocked 'connect' from '{uri}'", "Blocked inline 'connect'"), - # 'default-src': '', - 'font-src': (u"Blocked 'font' from '{uri}'", "Blocked inline 'font'"), - 'form-action': (u"Blocked 'form' action to '{uri}'", ), # no inline option - # 'frame-ancestors': '', - 'img-src': (u"Blocked 'image' from '{uri}'", "Blocked inline 'image'"), - 'manifest-src': (u"Blocked 'manifest' from '{uri}'", "Blocked inline 'manifest'"), - 'media-src': (u"Blocked 'media' from '{uri}'", "Blocked inline 'media'"), - 'object-src': (u"Blocked 'object' from '{uri}'", "Blocked inline 'object'"), - # 'plugin-types': '', - # 'referrer': '', - # 'reflected-xss': '', - 'script-src': (u"Blocked 'script' from '{uri}'", "Blocked unsafe (eval() or inline) 'script'"), - 'style-src': (u"Blocked 'style' from '{uri}'", "Blocked inline 'style'"), - # 'upgrade-insecure-requests': '', -} - -DEFAULT_MESSAGE = ('Blocked {directive!r} from {uri!r}', 'Blocked inline {directive!r}') - - -class Csp(Interface): - """ - A CSP violation report. - - See also: http://www.w3.org/TR/CSP/#violation-reports - - >>> { - >>> "document_uri": "http://example.com/", - >>> "violated_directive": "style-src cdn.example.com", - >>> "blocked_uri": "http://example.com/style.css", - >>> "effective_directive": "style-src", - >>> } - """ - - score = 1300 - display_score = 1300 - - @classmethod - def to_python(cls, data): - kwargs = {k: trim(data.get(k, None), 1024) for k in REPORT_KEYS} - - # Anything resulting from an "inline" whatever violation is either sent - # as 'self', or left off. In the case if it missing, we want to noramalize. - if not kwargs['blocked_uri']: - kwargs['blocked_uri'] = 'self' - - return cls(**kwargs) - - def get_hash(self): - directive = self.effective_directive - uri = self._normalized_blocked_uri - - # We want to distinguish between the different script-src - # violations that happen in - if _is_unsafe_script(directive, uri) and self.violated_directive: - if "'unsafe-inline'" in self.violated_directive: - uri = "'unsafe-inline'" - elif "'unsafe-eval'" in self.violated_directive: - uri = "'unsafe-eval'" - - return [directive, uri] - - def get_message(self): - directive = self.effective_directive - uri = self._normalized_blocked_uri - - index = 1 if uri == SELF else 0 - - tmpl = None - - # We want to special case script-src because they have - # unsafe-inline and unsafe-eval, but the report is ambiguous. - # so we want to attempt to guess which it was - if _is_unsafe_script(directive, uri) and self.violated_directive: - if "'unsafe-inline'" in self.violated_directive: - tmpl = "Blocked unsafe inline 'script'" - elif "'unsafe-eval'" in self.violated_directive: - tmpl = "Blocked unsafe eval() 'script'" - - if tmpl is None: - try: - tmpl = DIRECTIVE_TO_MESSAGES[directive][index] - except (KeyError, IndexError): - tmpl = DEFAULT_MESSAGE[index] - - return tmpl.format(directive=directive, uri=uri) - - def get_culprit(self): - return self._normalize_directive(self.violated_directive) - - def get_tags(self): - return [ - ('effective-directive', self.effective_directive), - ('blocked-uri', self.sanitized_blocked_uri()), - ] - - def sanitized_blocked_uri(self): - # HACK: This is 100% to work around Stripe urls - # that will casually put extremely sensitive information - # in querystrings. The real solution is to apply - # data scrubbing to all tags generically - uri = self.blocked_uri - if uri[:23] == 'https://api.stripe.com/': - return urlunsplit(urlsplit(uri)[:3] + (None, None)) - return uri - - @memoize - def _normalized_blocked_uri(self): - return _normalize_uri(self.blocked_uri) - - @memoize - def _normalized_document_uri(self): - return _normalize_uri(self.document_uri) - - def _normalize_directive(self, directive): - bits = [d for d in directive.split(' ') if d] - return ' '.join([bits[0]] + list(map(self._normalize_value, bits[1:]))) - - def _normalize_value(self, value): - # > If no scheme is specified, the same scheme as the one used to - # > access the protected document is assumed. - # Source: https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives - if value in KEYWORDS: - return value - - # normalize a value down to 'self' if it matches the origin of document-uri - # FireFox transforms a 'self' value into the spelled out origin, so we - # want to reverse this and bring it back - if value.startswith(ALL_SCHEMES): - if self._normalized_document_uri == _normalize_uri(value): - return SELF - # Their rule had an explicit scheme, so let's respect that - return value - - # value doesn't have a scheme, but let's see if their - # hostnames match at least, if so, they're the same - if value == self._normalized_document_uri: - return SELF - - # Now we need to stitch on a scheme to the value - scheme = self.document_uri.split(':', 1)[0] - # But let's not stitch on the boring values - if scheme in ('http', 'https'): - return value - return _unsplit(scheme, value) - - def get_title(self): - return 'CSP Report' - - def to_string(self, is_public=False, **kwargs): - return json.dumps({'csp-report': self.get_api_context()}, indent=2) - - def to_email_html(self, event, **kwargs): - return render_to_string( - 'sentry/partial/interfaces/csp_email.html', {'data': self.get_api_context()} - ) - - def get_path(self): - return 'sentry.interfaces.Csp' - - -def _is_unsafe_script(directive, uri): - return directive == 'script-src' and uri == SELF - - -def _normalize_uri(value): - if value in ('self', "'self'"): - return SELF - - # A lot of these values get reported as literally - # just the scheme. So a value like 'data' or 'blob', which - # are valid schemes, just not a uri. So we want to - # normalize it into a uri. - if ':' not in value: - scheme, hostname = value, '' - else: - scheme, hostname = urlsplit(value)[:2] - if scheme in ('http', 'https'): - return hostname - return _unsplit(scheme, hostname) - - -def _unsplit(scheme, hostname): - return urlunsplit((scheme, hostname, '', None, None)) diff --git a/src/sentry/interfaces/schemas.py b/src/sentry/interfaces/schemas.py index 8807c8970595d3..79bffda7dc121c 100644 --- a/src/sentry/interfaces/schemas.py +++ b/src/sentry/interfaces/schemas.py @@ -393,6 +393,189 @@ def apierror(message="Invalid data"): 'required': ['platform', 'event_id'], 'additionalProperties': True, } + +CSP_SCHEMA = { + 'type': 'object', + 'properties': { + 'csp-report': { + 'type': 'object', + 'properties': { + 'effective-directive': { + 'type': 'string', + 'enum': [ + 'base-uri', + 'child-src', + 'connect-src', + 'default-src', + 'font-src', + 'form-action', + 'frame-ancestors', + 'img-src', + 'manifest-src', + 'media-src', + 'object-src', + 'plugin-types', + 'referrer', + 'script-src', + 'style-src', + 'upgrade-insecure-requests', + # 'frame-src', # Deprecated (https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives#frame-src) + # 'sandbox', # Unsupported + ], + }, + 'blocked-uri': { + 'type': 'string', + 'default': 'self', + }, + 'document-uri': { + 'type': 'string', + 'not': {'enum': ['about:blank']} + }, + 'original-policy': {'type': 'string'}, + 'referrer': {'type': 'string', 'default': ''}, + 'status-code': {'type': 'number'}, + 'violated-directive': {'type': 'string', 'default': ''}, + 'source-file': {'type': 'string'}, + 'line-number': {'type': 'number'}, + 'column-number': {'type': 'number'}, + 'script-sample': {'type': 'number'}, # Firefox specific key. + }, + 'required': ['effective-directive'], + 'additionalProperties': False, # Don't allow any other keys. + } + }, + 'required': ['csp-report'], + 'additionalProperties': False, +} + +CSP_INTERFACE_SCHEMA = { + 'type': 'object', + 'properties': {k.replace('-', '_'): v for k, v in six.iteritems(CSP_SCHEMA['properties']['csp-report']['properties'])}, + 'required': ['effective_directive', 'violated_directive', 'blocked_uri'], + 'additionalProperties': False, # Don't allow any other keys. +} + +EXPECT_CT_SCHEMA = { + 'type': 'object', + 'properties': { + 'expect-ct-report': { + 'type': 'object', + 'properties': { + 'date-time': { + 'type': 'string', + 'format': 'date-time', + }, + 'hostname': {'type': 'string'}, + 'port': {'type': 'number'}, + 'effective-expiration-date': { + 'type': 'string', + 'format': 'date-time', + }, + 'served-certificate-chain': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'validated-certificate-chain': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'scts': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'version': {'type': 'number'}, + 'status': { + 'type': 'string', + 'enum': ['unknown', 'valid', 'invalid'], + }, + 'source': { + 'type': 'string', + 'enum': ['tls-extension', 'ocsp', 'embedded'], + }, + 'serialized_sct': {'type': 'string'}, # Base64 + }, + 'additionalProperties': False, + }, + }, + }, + 'required': ['hostname'], + 'additionalProperties': False, + }, + }, + 'additionalProperties': False, +} + +EXPECT_CT_INTERFACE_SCHEMA = { + 'type': 'object', + 'properties': {k.replace('-', '_'): v for k, v in + six.iteritems(EXPECT_CT_SCHEMA['properties']['expect-ct-report']['properties'])}, + 'required': ['hostname'], + 'additionalProperties': False, +} + +EXPECT_STAPLE_SCHEMA = { + 'type': 'object', + 'properties': { + 'expect-staple-report': { + 'type': 'object', + 'properties': { + 'date-time': { + 'type': 'string', + 'format': 'date-time', + }, + 'hostname': {'type': 'string'}, + 'port': {'type': 'number'}, + 'effective-expiration-date': { + 'type': 'string', + 'format': 'date-time', + }, + 'response-status': { + 'type': 'string', + 'enum': [ + 'MISSING', + 'PROVIDED', + 'ERROR_RESPONSE', + 'BAD_PRODUCED_AT', + 'NO_MATCHING_RESPONSE', + 'INVALID_DATE', + 'PARSE_RESPONSE_ERROR', + 'PARSE_RESPONSE_DATA_ERROR', + ], + }, + 'ocsp-response': {}, + 'cert-status': { + 'type': 'string', + 'enum': [ + 'GOOD', + 'REVOKED', + 'UNKNOWN', + ], + }, + 'served-certificate-chain': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'validated-certificate-chain': { + 'type': 'array', + 'items': {'type': 'string'} + }, + }, + 'required': ['hostname'], + 'additionalProperties': False, + }, + }, + 'additionalProperties': False, +} + +EXPECT_STAPLE_INTERFACE_SCHEMA = { + 'type': 'object', + 'properties': {k.replace('-', '_'): v for k, v in + six.iteritems(EXPECT_STAPLE_SCHEMA['properties']['expect-staple-report']['properties'])}, + 'required': ['hostname'], + 'additionalProperties': False, +} + """ Schemas for raw request data. @@ -400,6 +583,9 @@ def apierror(message="Invalid data"): then be transformed into the requisite interface. """ INPUT_SCHEMAS = { + 'sentry.interfaces.Csp': CSP_SCHEMA, + 'expectct': EXPECT_CT_SCHEMA, + 'expectstaple': EXPECT_STAPLE_SCHEMA, } """ @@ -409,7 +595,7 @@ def apierror(message="Invalid data"): should conform to these schemas. Currently this is not enforced everywhere yet. """ INTERFACE_SCHEMAS = { - # These should match SENTRY_INTERFACES keys + # Sentry interfaces 'sentry.interfaces.Http': HTTP_INTERFACE_SCHEMA, 'request': HTTP_INTERFACE_SCHEMA, 'exception': EXCEPTION_INTERFACE_SCHEMA, @@ -423,6 +609,11 @@ def apierror(message="Invalid data"): 'sentry.interfaces.Template': TEMPLATE_INTERFACE_SCHEMA, 'device': DEVICE_INTERFACE_SCHEMA, + # Security reports + 'sentry.interfaces.Csp': CSP_INTERFACE_SCHEMA, + 'expectct': EXPECT_CT_INTERFACE_SCHEMA, + 'expectstaple': EXPECT_STAPLE_INTERFACE_SCHEMA, + # Not interfaces per se, but looked up as if they were. 'event': EVENT_SCHEMA, 'tags': TAGS_TUPLES_SCHEMA, @@ -433,7 +624,11 @@ def apierror(message="Invalid data"): def validator_for_interface(name): if name not in INTERFACE_SCHEMAS: return None - return jsonschema.Draft4Validator(INTERFACE_SCHEMAS[name], types={'array': (list, tuple)}) + return jsonschema.Draft4Validator( + INTERFACE_SCHEMAS[name], + types={'array': (list, tuple)}, + format_checker=jsonschema.FormatChecker() + ) def validate_and_default_interface(data, interface, name=None, diff --git a/src/sentry/interfaces/security.py b/src/sentry/interfaces/security.py new file mode 100644 index 00000000000000..263cc398174284 --- /dev/null +++ b/src/sentry/interfaces/security.py @@ -0,0 +1,467 @@ +""" +sentry.interfaces.security +~~~~~~~~~~~~~~~~~~~~~ + +:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details. +:license: BSD, see LICENSE for more details. +""" + +from __future__ import absolute_import + +import jsonschema +import six + +__all__ = ('Csp', 'ExpectCT', 'ExpectStaple') + +from six.moves.urllib.parse import urlsplit, urlunsplit + +from sentry.interfaces.base import Interface, InterfaceValidationError +from sentry.interfaces.schemas import validate_and_default_interface, INPUT_SCHEMAS +from sentry.utils import json +from sentry.utils.cache import memoize +from sentry.utils.http import is_valid_origin +from sentry.utils.safe import trim +from sentry.web.helpers import render_to_string + +# Default block list sourced from personal experience as well as +# reputable blogs from Twitter and Dropbox +DEFAULT_DISALLOWED_SOURCES = ( + 'about', # Noise from Chrome about page. + 'ms-browser-extension', + 'chrome://*', + 'chrome-extension://*', + 'chromeinvokeimmediate://*' + 'chromenull://*', + 'safari-extension://*', + 'mxaddon-pkg://*', + 'jar://*', + 'webviewprogressproxy://*', + 'ms-browser-extension://*', + 'tmtbff://*', + 'mbinit://*', + 'symres://*', + 'resource://*', + '*.metrext.com', + 'static.image2play.com', + '*.tlscdn.com', + '73a5b0806e464be8bd4e694c744624f0.com', + '020dfefc4ac745dab7594f2f771c1ded.com', + '*.superfish.com', + 'addons.mozilla.org', + 'v.zilionfast.in', + 'widgets.amung.us', + '*.superfish.com', + 'xls.searchfun.in', + 'istatic.datafastguru.info', + 'v.zilionfast.in', + 'localhost', + 'resultshub-a.akamaihd.net', + 'pulseadnetwork.com', + 'gateway.zscalertwo.net', + 'www.passpack.com', + 'middlerush-a.akamaihd.net', + 'www.websmartcenter.com', + 'a.linkluster.com', + 'saveyoutime.ru', + 'cdncache-a.akamaihd.net', + 'x.rafomedia.com', + 'savingsslider-a.akamaihd.net', + 'injections.adguard.com', + 'icontent.us', + 'amiok.org', + 'connectionstrenth.com', + 'siteheart.net', + 'netanalitics.space', +) # yapf: disable + + +class SecurityReport(Interface): + """ + A browser security violation report. + """ + + path = None + title = None + + @classmethod + def from_raw(cls, raw): + """ + Constructs the interface from a raw security report request body + + This is usually slightly different than to_python as it needs to + do some extra validation, data extraction / default setting. + """ + raise NotImplementedError + + @classmethod + def to_python(cls, data): + is_valid, errors = validate_and_default_interface(data, cls.path) + if not is_valid: + raise InterfaceValidationError("Invalid interface data") + + return cls(**data) + + def get_culprit(self): + raise NotImplementedError + + def get_message(self): + raise NotImplementedError + + def get_path(self): + return self.path + + def get_tags(self): + raise NotImplementedError + + def get_title(self): + return self.title + + def should_filter(self, project=None): + raise NotImplementedError + + def get_origin(self): + """ + The document URL that generated this report + """ + raise NotImplementedError + + def get_referrer(self): + """ + The referrer of the page that generated this report. + """ + raise NotImplementedError + + +class ExpectStaple(SecurityReport): + """ + An OCSP Stapling violation report + + See: https://docs.google.com/document/d/1aISglJIIwglcOAhqNfK-2vtQl-_dWAapc-VLDh-9-BE + >>> { + >>> "date-time": date-time, + >>> "hostname": hostname, + >>> "port": port, + >>> "effective-expiration-date": date-time, + >>> "response-status": ResponseStatus, + >>> "ocsp-response": ocsp, + >>> "cert-status": CertStatus, + >>> "served-certificate-chain": [pem1, ... pemN],(MUST be in the order served) + >>> "validated-certificate-chain": [pem1, ... pemN](MUST be in the order served) + >>> } + """ + + score = 1300 + display_score = 1300 + + path = 'expectstaple' + title = 'Expect-Staple Report' + + @classmethod + def from_raw(cls, raw): + # Validate the raw data against the input schema (raises on failure) + schema = INPUT_SCHEMAS[cls.path] + jsonschema.validate(raw, schema) + + # For Expect-Staple, the values we want are nested under the + # 'expect-staple-report' key. + raw = raw['expect-staple-report'] + # Trim values and convert keys to use underscores + kwargs = {k.replace('-', '_'): trim(v, 1024) for k, v in six.iteritems(raw)} + + return cls.to_python(kwargs) + + def get_culprit(self): + return self.hostname + + def get_hash(self, is_processed_data=True): + return [self.hostname] + + def get_message(self): + return "Expect-Staple failed for '{self.hostname}'".format(self=self) + + def get_tags(self): + return ( + ('port', six.text_type(self.port)), + ('hostname', self.hostname), + ('response_status', self.response_status), + ('cert_status', self.cert_status), + ) + + def get_origin(self): + return self.hostname + + def get_referrer(self): + return None + + def should_filter(self, project=None): + return False + + +class ExpectCT(SecurityReport): + """ + A Certificate Transparency violation report. + + See also: http://httpwg.org/http-extensions/expect-ct.html + >>> { + >>> "date-time": "2014-04-06T13:00:50Z", + >>> "hostname": "www.example.com", + >>> "port": 443, + >>> "effective-expiration-date": "2014-05-01T12:40:50Z", + >>> "served-certificate-chain": [], + >>> "validated-certificate-chain": [], + >>> "scts-pins": [], + >>> } + """ + + score = 1300 + display_score = 1300 + + path = 'expectct' + title = 'Expect-CT Report' + + @classmethod + def from_raw(cls, raw): + # Validate the raw data against the input schema (raises on failure) + schema = INPUT_SCHEMAS[cls.path] + jsonschema.validate(raw, schema) + + # For Expect-CT, the values we want are nested under the 'expect-ct-report' key. + raw = raw['expect-ct-report'] + # Trim values and convert keys to use underscores + kwargs = {k.replace('-', '_'): trim(v, 1024) for k, v in six.iteritems(raw)} + + return cls.to_python(kwargs) + + def get_culprit(self): + return self.hostname + + def get_hash(self, is_processed_data=True): + return [self.hostname] + + def get_message(self): + return "Expect-CT failed for '{self.hostname}'".format(self=self) + + def get_tags(self): + return ( + ('port', six.text_type(self.port)), + ('hostname', self.hostname), + ) + + def get_origin(self): + return self.hostname # not quite origin, but the domain that failed pinning + + def get_referrer(self): + return None + + def should_filter(self, project=None): + return False + + +class Csp(SecurityReport): + """ + A CSP violation report. + + See also: http://www.w3.org/TR/CSP/#violation-reports + + >>> { + >>> "document_uri": "http://example.com/", + >>> "violated_directive": "style-src cdn.example.com", + >>> "blocked_uri": "http://example.com/style.css", + >>> "effective_directive": "style-src", + >>> } + """ + + LOCAL = "'self'" + score = 1300 + display_score = 1300 + + path = 'sentry.interfaces.Csp' + title = 'CSP Report' + + @classmethod + def from_raw(cls, raw): + # Validate the raw data against the input schema (raises on failure) + schema = INPUT_SCHEMAS[cls.path] + jsonschema.validate(raw, schema) + + # For CSP, the values we want are nested under the 'csp-report' key. + raw = raw['csp-report'] + # Trim values and convert keys to use underscores + kwargs = {k.replace('-', '_'): trim(v, 1024) for k, v in six.iteritems(raw)} + + return cls.to_python(kwargs) + + def get_hash(self, is_processed_data=True): + if self._local_script_violation_type: + uri = "'%s'" % self._local_script_violation_type + else: + uri = self._normalized_blocked_uri + + return [self.effective_directive, uri] + + def get_message(self): + templates = { + 'child-src': (u"Blocked 'child' from '{uri}'", "Blocked inline 'child'"), + 'connect-src': (u"Blocked 'connect' from '{uri}'", "Blocked inline 'connect'"), + 'font-src': (u"Blocked 'font' from '{uri}'", "Blocked inline 'font'"), + 'form-action': (u"Blocked 'form' action to '{uri}'", ), # no inline option + 'img-src': (u"Blocked 'image' from '{uri}'", "Blocked inline 'image'"), + 'manifest-src': (u"Blocked 'manifest' from '{uri}'", "Blocked inline 'manifest'"), + 'media-src': (u"Blocked 'media' from '{uri}'", "Blocked inline 'media'"), + 'object-src': (u"Blocked 'object' from '{uri}'", "Blocked inline 'object'"), + 'script-src': (u"Blocked 'script' from '{uri}'", "Blocked unsafe (eval() or inline) 'script'"), + 'style-src': (u"Blocked 'style' from '{uri}'", "Blocked inline 'style'"), + 'unsafe-inline': (None, u"Blocked unsafe inline 'script'"), + 'unsafe-eval': (None, u"Blocked unsafe eval() 'script'"), + } + default_template = ('Blocked {directive!r} from {uri!r}', 'Blocked inline {directive!r}') + + directive = self._local_script_violation_type or self.effective_directive + uri = self._normalized_blocked_uri + index = 1 if uri == self.LOCAL else 0 + + try: + tmpl = templates[directive][index] + except (KeyError, IndexError): + tmpl = default_template[index] + + return tmpl.format(directive=directive, uri=uri) + + def get_culprit(self): + if not self.violated_directive: + return '' + bits = [d for d in self.violated_directive.split(' ') if d] + return ' '.join([bits[0]] + [self._normalize_value(b) for b in bits[1:]]) + + def get_tags(self): + return [ + ('effective-directive', self.effective_directive), + ('blocked-uri', self._sanitized_blocked_uri()), + ] + + def get_origin(self): + return self.document_uri + + def get_referrer(self): + return self.referrer + + def to_string(self, is_public=False, **kwargs): + return json.dumps({'csp-report': self.get_api_context()}, indent=2) + + def to_email_html(self, event, **kwargs): + return render_to_string( + 'sentry/partial/interfaces/csp_email.html', {'data': self.get_api_context()} + ) + + def should_filter(self, project=None): + disallowed = () + paths = ['blocked_uri', 'source_file'] + uris = [getattr(self, path) for path in paths if hasattr(self, path)] + + if project is None or bool(project.get_option('sentry:csp_ignored_sources_defaults', True)): + disallowed += DEFAULT_DISALLOWED_SOURCES + if project is not None: + disallowed += tuple(project.get_option('sentry:csp_ignored_sources', [])) + + if disallowed and any(is_valid_origin(uri and uri, allowed=disallowed) for uri in uris): + return True + + return False + + def _sanitized_blocked_uri(self): + # HACK: This is 100% to work around Stripe urls + # that will casually put extremely sensitive information + # in querystrings. The real solution is to apply + # data scrubbing to all tags generically + # TODO this could be done in filter_csp + # instead but that might only be run conditionally on the org/project settings + # relevant code is @L191: + # + # if netloc == 'api.stripe.com': + # query = '' + # fragment = '' + + uri = self.blocked_uri + if uri.startswith('https://api.stripe.com/'): + return urlunsplit(urlsplit(uri)[:3] + (None, None)) + return uri + + @memoize + def _normalized_blocked_uri(self): + return self._normalize_uri(self.blocked_uri) + + @memoize + def _normalized_document_uri(self): + return self._normalize_uri(self.document_uri) + + def _normalize_value(self, value): + keywords = ("'none'", "'self'", "'unsafe-inline'", "'unsafe-eval'", ) + all_schemes = ( + 'data:', + 'mediastream:', + 'blob:', + 'filesystem:', + 'http:', + 'https:', + 'file:', + ) + + # > If no scheme is specified, the same scheme as the one used to + # > access the protected document is assumed. + # Source: https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives + if value in keywords: + return value + + # normalize a value down to 'self' if it matches the origin of document-uri + # FireFox transforms a 'self' value into the spelled out origin, so we + # want to reverse this and bring it back + if value.startswith(all_schemes): + if self._normalized_document_uri == self._normalize_uri(value): + return self.LOCAL + # Their rule had an explicit scheme, so let's respect that + return value + + # value doesn't have a scheme, but let's see if their + # hostnames match at least, if so, they're the same + if value == self._normalized_document_uri: + return self.LOCAL + + # Now we need to stitch on a scheme to the value + scheme = self.document_uri.split(':', 1)[0] + # But let's not stitch on the boring values + if scheme in ('http', 'https'): + return value + return self._unsplit(scheme, value) + + @memoize + def _local_script_violation_type(self): + """ + If this is a locally-sourced script-src error, gives the type. + """ + if (self.violated_directive + and self.effective_directive == 'script-src' + and self._normalized_blocked_uri == self.LOCAL): + if "'unsafe-inline'" in self.violated_directive: + return "unsafe-inline" + elif "'unsafe-eval'" in self.violated_directive: + return "unsafe-eval" + return None + + def _normalize_uri(self, value): + if value in ('', self.LOCAL, self.LOCAL.strip("'")): + return self.LOCAL + + # A lot of these values get reported as literally + # just the scheme. So a value like 'data' or 'blob', which + # are valid schemes, just not a uri. So we want to + # normalize it into a uri. + if ':' not in value: + scheme, hostname = value, '' + else: + scheme, hostname = urlsplit(value)[:2] + if scheme in ('http', 'https'): + return hostname + return self._unsplit(scheme, hostname) + + def _unsplit(self, scheme, hostname): + return urlunsplit((scheme, hostname, '', None, None)) diff --git a/src/sentry/utils/csp.py b/src/sentry/utils/csp.py deleted file mode 100644 index d689c211a0e050..00000000000000 --- a/src/sentry/utils/csp.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -sentry.utils.csp -~~~~~~~~~~~~~~~~ - -:copyright: (c) 2010-2015 by the Sentry Team, see AUTHORS for more details. -:license: BSD, see LICENSE for more details. -""" -from __future__ import absolute_import - -from sentry.utils.http import is_valid_origin - -# Default block list sourced from personal experience as well as -# reputable blogs from Twitter and Dropbox -DISALLOWED_SOURCES = ( - 'chrome://*', - 'chrome-extension://*', - 'chromeinvokeimmediate://*' - 'chromenull://*', - 'safari-extension://*', - 'mxaddon-pkg://*', - 'jar://*', - 'webviewprogressproxy://*', - 'ms-browser-extension://*', - 'tmtbff://*', - 'mbinit://*', - 'symres://*', - 'resource://*', - '*.metrext.com', - 'static.image2play.com', - '*.tlscdn.com', - '73a5b0806e464be8bd4e694c744624f0.com', - '020dfefc4ac745dab7594f2f771c1ded.com', - '*.superfish.com', - 'addons.mozilla.org', - 'v.zilionfast.in', - 'widgets.amung.us', - '*.superfish.com', - 'xls.searchfun.in', - 'istatic.datafastguru.info', - 'v.zilionfast.in', - 'localhost', - 'resultshub-a.akamaihd.net', - 'pulseadnetwork.com', - 'gateway.zscalertwo.net', - 'www.passpack.com', - 'middlerush-a.akamaihd.net', - 'www.websmartcenter.com', - 'a.linkluster.com', - 'saveyoutime.ru', - 'cdncache-a.akamaihd.net', - 'x.rafomedia.com', - 'savingsslider-a.akamaihd.net', - 'injections.adguard.com', - 'icontent.us', - 'amiok.org', - 'connectionstrenth.com', - 'siteheart.net', - 'netanalitics.space', -) # yapf: disable - -ALLOWED_DIRECTIVES = frozenset(( - 'base-uri', - 'child-src', - 'connect-src', - 'default-src', - 'font-src', - 'form-action', - 'frame-ancestors', - 'img-src', - 'manifest-src', - 'media-src', - 'object-src', - 'plugin-types', - 'referrer', - 'script-src', - 'style-src', - 'upgrade-insecure-requests', - - # Deprecated directives - # > Note: This directive is deprecated. Use child-src instead. - # > https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives#frame-src - # 'frame-src', - - # I don't really know what this even is. - # 'sandbox', -)) # yapf: disable - -# URIs that are pure noise and will never be actionable -DISALLOWED_BLOCKED_URIS = frozenset(( - 'about', - 'ms-browser-extension', -)) # yapf: disable - - -def is_valid_csp_report(report, project=None): - # Some reports from Chrome report blocked-uri as just 'about'. - # In this case, this is not actionable and is just noisy. - # Observed in Chrome 45 and 46. - if report.get('effective_directive') not in ALLOWED_DIRECTIVES: - return False - - blocked_uri = report.get('blocked_uri') - if blocked_uri in DISALLOWED_BLOCKED_URIS: - return False - - source_file = report.get('source_file') - - # We must have one of these to do anyting sensible - if not any((blocked_uri, source_file)): - return False - - if project is None or bool(project.get_option('sentry:csp_ignored_sources_defaults', True)): - disallowed_sources = DISALLOWED_SOURCES - else: - disallowed_sources = () - - if project is not None: - disallowed_sources += tuple(project.get_option('sentry:csp_ignored_sources', [])) - - if not disallowed_sources: - return True - - if source_file and is_valid_origin(source_file, allowed=disallowed_sources): - return False - - if blocked_uri and is_valid_origin(blocked_uri, allowed=disallowed_sources): - return False - - return True diff --git a/src/sentry/web/api.py b/src/sentry/web/api.py index 0ba8a6bc36052e..ff021d9d8dd0a5 100644 --- a/src/sentry/web/api.py +++ b/src/sentry/web/api.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, print_function import base64 +import jsonschema import logging import six import traceback @@ -22,10 +23,11 @@ from sentry import quotas, tsdb from sentry.coreapi import ( - APIError, APIForbidden, APIRateLimited, ClientApiHelper, CspApiHelper, LazyData, + APIError, APIForbidden, APIRateLimited, ClientApiHelper, SecurityApiHelper, LazyData, MinidumpApiHelper, ) from sentry.interfaces import schemas +from sentry.interfaces.base import get_interface from sentry.lang.native.utils import merge_minidump_event from sentry.models import Project, OrganizationOption, Organization from sentry.signals import ( @@ -452,12 +454,6 @@ def process(self, request, project, key, auth, helper, data, **kwargs): org_options = OrganizationOption.objects.get_all_values( project.organization_id) - if org_options.get('sentry:require_scrub_ip_address', False): - scrub_ip_address = True - else: - scrub_ip_address = project.get_option( - 'sentry:scrub_ip_address', False) - event_id = data['event_id'] # TODO(dcramer): ideally we'd only validate this if the event_id was @@ -468,10 +464,10 @@ def process(self, request, project, key, auth, helper, data, **kwargs): raise APIForbidden( 'An event with the same ID already exists (%s)' % (event_id, )) - if org_options.get('sentry:require_scrub_data', False): - scrub_data = True - else: - scrub_data = project.get_option('sentry:scrub_data', True) + scrub_ip_address = (org_options.get('sentry:require_scrub_ip_address', False) or + project.get_option('sentry:scrub_ip_address', False)) + scrub_data = (org_options.get('sentry:require_scrub_data', False) or + project.get_option('sentry:scrub_data', True)) if scrub_data: # We filter data immediately before it ever gets into the queue @@ -487,18 +483,14 @@ def process(self, request, project, key, auth, helper, data, **kwargs): project.get_option(exclude_fields_key, []) ) - if org_options.get('sentry:require_scrub_defaults', False): - scrub_defaults = True - else: - scrub_defaults = project.get_option( - 'sentry:scrub_defaults', True) + scrub_defaults = (org_options.get('sentry:require_scrub_defaults', False) or + project.get_option('sentry:scrub_defaults', True)) - inst = SensitiveDataFilter( + SensitiveDataFilter( fields=sensitive_fields, include_defaults=scrub_defaults, exclude_fields=exclude_fields, - ) - inst.apply(data) + ).apply(data) if scrub_ip_address: # We filter data immediately before it ever gets into the queue @@ -602,9 +594,15 @@ def get(self, request, **kwargs): return HttpResponse(json.dumps(schemas.EVENT_SCHEMA), content_type='application/json') -class CspReportView(StoreView): - helper_cls = CspApiHelper - content_types = ('application/csp-report', 'application/json') +class SecurityReportView(StoreView): + helper_cls = SecurityApiHelper + content_types = ( + 'application/csp-report', + 'application/json', + 'application/expect-ct-report', + 'application/expect-ct-report+json', + 'application/expect-staple-report', + ) def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): # A CSP report is sent as a POST request with no Origin or Referer @@ -642,40 +640,49 @@ def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwar ) def post(self, request, project, helper, **kwargs): - data = helper.safely_load_json_string(request.body) + json_body = helper.safely_load_json_string(request.body) + report_type = self.security_report_type(json_body) + if report_type is None: + raise APIError('Unrecognized security report type') + interface = get_interface(report_type) - # Do origin check based on the `document-uri` key as explained - # in `_dispatch`. try: - report = data['csp-report'] - except KeyError: - raise APIError('Missing csp-report') - - origin = report.get('document-uri') - - # No idea, but this is garbage - if origin == 'about:blank': - raise APIForbidden('Invalid document-uri') + instance = interface.from_raw(json_body) + except jsonschema.ValidationError as e: + raise APIError('Invalid security report: %s' % str(e).splitlines()[0]) + # Do origin check based on the `document-uri` key as explained in `_dispatch`. + origin = instance.get_origin() if not is_valid_origin(origin, project): if project: - tsdb.incr(tsdb.models.project_total_received_cors, - project.id) - raise APIForbidden('Invalid document-uri') + tsdb.incr(tsdb.models.project_total_received_cors, project.id) + raise APIForbidden('Invalid origin') - # Attach on collected meta data. This data obviously isn't a part - # of the spec, but we need to append to the report sentry specific things. - report['_meta'] = { + data = { + 'interface': interface.path, + 'report': instance, 'release': request.GET.get('sentry_release'), } response_or_event_id = self.process( - request, project=project, helper=helper, data=report, **kwargs + request, project=project, helper=helper, data=data, **kwargs ) if isinstance(response_or_event_id, HttpResponse): return response_or_event_id return HttpResponse(status=201) + def security_report_type(self, body): + report_type_for_key = { + 'csp-report': 'sentry.interfaces.Csp', + 'expect-ct-report': 'expectct', + 'expect-staple-report': 'expectstaple', + } + if isinstance(body, dict): + for k in report_type_for_key: + if k in body: + return report_type_for_key[k] + return None + @cache_control(max_age=3600, public=True) def robots_txt(request): diff --git a/src/sentry/web/urls.py b/src/sentry/web/urls.py index 0ef226f1488822..2d2350693f5f20 100644 --- a/src/sentry/web/urls.py +++ b/src/sentry/web/urls.py @@ -103,8 +103,13 @@ def init_all_applications(): name='sentry-api-minidump' ), url( + r'^api/(?P\d+)/security/$', + api.SecurityReportView.as_view(), + name='sentry-api-security-report' + ), + url( # This URL to be deprecated r'^api/(?P\d+)/csp-report/$', - api.CspReportView.as_view(), + api.SecurityReportView.as_view(), name='sentry-api-csp-report' ), url( diff --git a/tests/integration/tests.py b/tests/integration/tests.py index ef64f10f19810f..436d4db00935e4 100644 --- a/tests/integration/tests.py +++ b/tests/integration/tests.py @@ -483,7 +483,7 @@ def assertReportCreated(self, input, output): def assertReportRejected(self, input): resp = self._postCspWithHeader(input) - assert resp.status_code == 403, resp.content + assert resp.status_code in (400, 403), resp.content def test_chrome_blocked_asset(self): self.assertReportCreated(*get_fixtures('chrome_blocked_asset')) diff --git a/tests/sentry/coreapi/tests.py b/tests/sentry/coreapi/tests.py index d1cd513b8dab26..5da5428e420797 100644 --- a/tests/sentry/coreapi/tests.py +++ b/tests/sentry/coreapi/tests.py @@ -15,8 +15,7 @@ APIUnauthorized, Auth, ClientApiHelper, - CspApiHelper, - APIForbidden, + SecurityApiHelper, ) from sentry.event_manager import EventManager from sentry.interfaces.base import get_interface @@ -621,27 +620,23 @@ def test_with_all_auto_ip(self): assert out['sentry.interfaces.User']['ip_address'] == '127.0.0.1' -class CspApiHelperTest(BaseAPITest): - helper_cls = CspApiHelper +class SecurityApiHelperTest(BaseAPITest): + helper_cls = SecurityApiHelper - def test_validate_basic(self): + def test_csp_validate_basic(self): report = { - "document-uri": - "http://45.55.25.245:8123/csp", - "referrer": - "http://example.com", - "violated-directive": - "img-src https://45.55.25.245:8123/", - "effective-directive": - "img-src", - "original-policy": - "default-src https://45.55.25.245:8123/; child-src https://45.55.25.245:8123/; connect-src https://45.55.25.245:8123/; font-src https://45.55.25.245:8123/; img-src https://45.55.25.245:8123/; media-src https://45.55.25.245:8123/; object-src https://45.55.25.245:8123/; script-src https://45.55.25.245:8123/; style-src https://45.55.25.245:8123/; form-action https://45.55.25.245:8123/; frame-ancestors 'none'; plugin-types 'none'; report-uri http://45.55.25.245:8123/csp-report?os=OS%20X&device=&browser_version=43.0&browser=chrome&os_version=Lion", - "blocked-uri": - "http://google.com", - "status-code": - 200, - "_meta": { - "release": "abc123", + "release": "abc123", + "interface": 'sentry.interfaces.Csp', + "report": { + "csp-report": { + "document-uri": "http://45.55.25.245:8123/csp", + "referrer": "http://example.com", + "violated-directive": "img-src https://45.55.25.245:8123/", + "effective-directive": "img-src", + "original-policy": "default-src https://45.55.25.245:8123/; child-src https://45.55.25.245:8123/; connect-src https://45.55.25.245:8123/; font-src https://45.55.25.245:8123/; img-src https://45.55.25.245:8123/; media-src https://45.55.25.245:8123/; object-src https://45.55.25.245:8123/; script-src https://45.55.25.245:8123/; style-src https://45.55.25.245:8123/; form-action https://45.55.25.245:8123/; frame-ancestors 'none'; plugin-types 'none'; report-uri http://45.55.25.245:8123/csp-report?os=OS%20X&device=&browser_version=43.0&browser=chrome&os_version=Lion", + "blocked-uri": "http://google.com", + "status-code": 200, + } } } result = self.validate_and_normalize(report) @@ -661,29 +656,32 @@ def test_validate_basic(self): 'Referer': 'http://example.com' } - @mock.patch('sentry.interfaces.csp.Csp.to_python', mock.Mock(side_effect=Exception)) - def test_validate_raises_invalid_interface(self): - with self.assertRaises(APIForbidden): + def test_csp_validate_failure(self): + report = { + "release": "abc123", + "interface": 'sentry.interfaces.Csp', + "report": {} + } + with self.assertRaises(APIError): + self.validate_and_normalize(report) + + with self.assertRaises(APIError): self.validate_and_normalize({}) - def test_tags_out_of_bounds(self): + def test_csp_tags_out_of_bounds(self): report = { - "document-uri": - "http://45.55.25.245:8123/csp", - "referrer": - "http://example.com", - "violated-directive": - "img-src https://45.55.25.245:8123/", - "effective-directive": - "img-src", - "original-policy": - "default-src https://45.55.25.245:8123/; child-src https://45.55.25.245:8123/; connect-src https://45.55.25.245:8123/; font-src https://45.55.25.245:8123/; img-src https://45.55.25.245:8123/; media-src https://45.55.25.245:8123/; object-src https://45.55.25.245:8123/; script-src https://45.55.25.245:8123/; style-src https://45.55.25.245:8123/; form-action https://45.55.25.245:8123/; frame-ancestors 'none'; plugin-types 'none'; report-uri http://45.55.25.245:8123/csp-report?os=OS%20X&device=&browser_version=43.0&browser=chrome&os_version=Lion", - "blocked-uri": - "v" * 201, - "status-code": - 200, - "_meta": { - "release": "abc123", + "release": "abc123", + "interface": 'sentry.interfaces.Csp', + "report": { + "csp-report": { + "document-uri": "http://45.55.25.245:8123/csp", + "referrer": "http://example.com", + "violated-directive": "img-src https://45.55.25.245:8123/", + "effective-directive": "img-src", + "original-policy": "default-src https://45.55.25.245:8123/; child-src https://45.55.25.245:8123/; connect-src https://45.55.25.245:8123/; font-src https://45.55.25.245:8123/; img-src https://45.55.25.245:8123/; media-src https://45.55.25.245:8123/; object-src https://45.55.25.245:8123/; script-src https://45.55.25.245:8123/; style-src https://45.55.25.245:8123/; form-action https://45.55.25.245:8123/; frame-ancestors 'none'; plugin-types 'none'; report-uri http://45.55.25.245:8123/csp-report?os=OS%20X&device=&browser_version=43.0&browser=chrome&os_version=Lion", + "blocked-uri": "v" * 201, + "status-code": 200, + } } } result = self.validate_and_normalize(report) @@ -692,24 +690,20 @@ def test_tags_out_of_bounds(self): ] assert len(result['errors']) == 1 - def test_tag_value(self): + def test_csp_tag_value(self): report = { - "document-uri": - "http://45.55.25.245:8123/csp", - "referrer": - "http://example.com", - "violated-directive": - "img-src https://45.55.25.245:8123/", - "effective-directive": - "img-src", - "original-policy": - "default-src https://45.55.25.245:8123/; child-src https://45.55.25.245:8123/; connect-src https://45.55.25.245:8123/; font-src https://45.55.25.245:8123/; img-src https://45.55.25.245:8123/; media-src https://45.55.25.245:8123/; object-src https://45.55.25.245:8123/; script-src https://45.55.25.245:8123/; style-src https://45.55.25.245:8123/; form-action https://45.55.25.245:8123/; frame-ancestors 'none'; plugin-types 'none'; report-uri http://45.55.25.245:8123/csp-report?os=OS%20X&device=&browser_version=43.0&browser=chrome&os_version=Lion", - "blocked-uri": - "http://google.com", - "status-code": - 200, - "_meta": { - "release": "abc123", + "release": "abc123", + "interface": 'sentry.interfaces.Csp', + "report": { + "csp-report": { + "document-uri": "http://45.55.25.245:8123/csp", + "referrer": "http://example.com", + "violated-directive": "img-src https://45.55.25.245:8123/", + "effective-directive": "img-src", + "original-policy": "default-src https://45.55.25.245:8123/; child-src https://45.55.25.245:8123/; connect-src https://45.55.25.245:8123/; font-src https://45.55.25.245:8123/; img-src https://45.55.25.245:8123/; media-src https://45.55.25.245:8123/; object-src https://45.55.25.245:8123/; script-src https://45.55.25.245:8123/; style-src https://45.55.25.245:8123/; form-action https://45.55.25.245:8123/; frame-ancestors 'none'; plugin-types 'none'; report-uri http://45.55.25.245:8123/csp-report?os=OS%20X&device=&browser_version=43.0&browser=chrome&os_version=Lion", + "blocked-uri": "http://google.com", + "status-code": 200, + } } } result = self.validate_and_normalize(report) @@ -718,27 +712,3 @@ def test_tag_value(self): ('blocked-uri', 'http://google.com'), ] assert len(result['errors']) == 0 - - def test_no_tags(self): - report = { - "document-uri": - "http://45.55.25.245:8123/csp", - "referrer": - "http://example.com", - "violated-directive": - "img-src https://45.55.25.245:8123/", - "effective-directive": - "v" * 201, - "original-policy": - "default-src https://45.55.25.245:8123/; child-src https://45.55.25.245:8123/; connect-src https://45.55.25.245:8123/; font-src https://45.55.25.245:8123/; img-src https://45.55.25.245:8123/; media-src https://45.55.25.245:8123/; object-src https://45.55.25.245:8123/; script-src https://45.55.25.245:8123/; style-src https://45.55.25.245:8123/; form-action https://45.55.25.245:8123/; frame-ancestors 'none'; plugin-types 'none'; report-uri http://45.55.25.245:8123/csp-report?os=OS%20X&device=&browser_version=43.0&browser=chrome&os_version=Lion", - "blocked-uri": - "http://goo\ngle.com", - "status-code": - 200, - "_meta": { - "release": "abc123", - } - } - result = self.validate_and_normalize(report) - assert result['tags'] == [] - assert len(result['errors']) == 2 diff --git a/tests/sentry/interfaces/test_csp.py b/tests/sentry/interfaces/test_security.py similarity index 64% rename from tests/sentry/interfaces/test_csp.py rename to tests/sentry/interfaces/test_security.py index c5c0873535700f..ad135c723ec67f 100644 --- a/tests/sentry/interfaces/test_csp.py +++ b/tests/sentry/interfaces/test_security.py @@ -4,7 +4,7 @@ from exam import fixture -from sentry.interfaces.csp import Csp +from sentry.interfaces.security import Csp, ExpectCT, ExpectStaple from sentry.testutils import TestCase @@ -226,3 +226,107 @@ def test_get_message(self): ) ) assert result.get_message() == "Blocked 'script' from 'data:'" + + +class ExpectCTTest(TestCase): + + raw_report = { + "expect-ct-report": { + "date-time": "2014-04-06T13:00:50Z", + "hostname": "www.example.com", + "port": 443, + "effective-expiration-date": "2014-05-01T12:40:50Z", + "served-certificate-chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + "validated-certificate-chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + "scts": [ + { + "version": 1, + "status": "invalid", + "source": "embedded", + "serialized_sct": "ABCD==" + }, + ], + } + } + interface_json = { + 'date_time': '2014-04-06T13:00:50Z', + 'hostname': 'www.example.com', + 'port': 443, + 'effective_expiration_date': '2014-05-01T12:40:50Z', + 'served_certificate_chain': ['-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----'], + 'validated_certificate_chain': ['-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----'], + 'scts': [{ + 'status': 'invalid', + 'source': 'embedded', + 'serialized_sct': 'ABCD==', + 'version': 1 + }] + } + + def test_from_raw(self): + interface = ExpectCT.from_raw(self.raw_report) + assert interface.hostname == 'www.example.com' + assert interface.date_time == '2014-04-06T13:00:50Z' + assert interface.port == 443 + assert len(interface.served_certificate_chain) == 1 + + def test_to_python(self): + interface = ExpectCT.to_python(self.interface_json) + assert interface.hostname == 'www.example.com' + assert interface.date_time == '2014-04-06T13:00:50Z' + assert interface.port == 443 + assert len(interface.served_certificate_chain) == 1 + + def test_serialize_unserialize_behavior(self): + assert ExpectCT.to_python(self.interface_json).to_json() == self.interface_json + + def test_invalid_format(self): + interface = ExpectCT.to_python({ + 'hostname': 'www.example.com', + 'date_time': 'Not an RFC3339 datetime' + }) + # invalid keys are just removed + assert interface.to_json() == {'hostname': 'www.example.com'} + + +class ExpectStapleTest(TestCase): + + raw_report = { + "expect-staple-report": { + "date-time": "2014-04-06T13:00:50Z", + "hostname": "www.example.com", + "port": 443, + "response-status": "ERROR_RESPONSE", + "cert-status": "REVOKED", + "effective-expiration-date": "2014-05-01T12:40:50Z", + "served-certificate-chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + "validated-certificate-chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + } + } + interface_json = { + "date_time": "2014-04-06T13:00:50Z", + "hostname": "www.example.com", + "port": 443, + "response_status": "ERROR_RESPONSE", + "cert_status": "REVOKED", + "effective_expiration_date": "2014-05-01T12:40:50Z", + "served_certificate_chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + "validated_certificate_chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + } + + def test_from_raw(self): + interface = ExpectStaple.from_raw(self.raw_report) + assert interface.hostname == 'www.example.com' + assert interface.date_time == '2014-04-06T13:00:50Z' + assert interface.port == 443 + assert len(interface.served_certificate_chain) == 1 + + def test_to_python(self): + interface = ExpectStaple.to_python(self.interface_json) + assert interface.hostname == 'www.example.com' + assert interface.date_time == '2014-04-06T13:00:50Z' + assert interface.port == 443 + assert len(interface.served_certificate_chain) == 1 + + def test_serialize_unserialize_behavior(self): + assert ExpectStaple.to_python(self.interface_json).to_json() == self.interface_json diff --git a/tests/sentry/utils/test_csp.py b/tests/sentry/utils/test_csp.py index b22a738ab63e3d..1baf226ccdb05e 100644 --- a/tests/sentry/utils/test_csp.py +++ b/tests/sentry/utils/test_csp.py @@ -2,16 +2,25 @@ import pytest -from sentry.utils.csp import is_valid_csp_report +from sentry.interfaces.base import InterfaceValidationError +from sentry.interfaces.security import Csp @pytest.mark.parametrize( 'report', ( {}, { 'effective_directive': 'lolnotreal' - }, { - 'effective_directive': 'style-src' - }, { + }, + ) +) +def test_invalid_csp_report(report): + with pytest.raises(InterfaceValidationError): + Csp.to_python(report) + + +@pytest.mark.parametrize( + 'report', ( + { 'effective_directive': 'style-src', 'blocked_uri': 'about' }, { @@ -36,7 +45,7 @@ ) ) def test_blocked_csp_report(report): - assert is_valid_csp_report(report) is False + assert Csp.to_python(report).should_filter() is True @pytest.mark.parametrize( @@ -50,8 +59,10 @@ def test_blocked_csp_report(report): }, { 'effective_directive': 'style-src', 'source_file': 'http://example.com' + }, { + 'effective_directive': 'style-src' }, ) ) def test_valid_csp_report(report): - assert is_valid_csp_report(report) is True + assert Csp.to_python(report).should_filter() is False diff --git a/tests/sentry/web/api/tests.py b/tests/sentry/web/api/tests.py index 917656aee9feb7..e99bb741d4db99 100644 --- a/tests/sentry/web/api/tests.py +++ b/tests/sentry/web/api/tests.py @@ -15,10 +15,10 @@ from sentry.utils.data_filters import FilterTypes -class CspReportViewTest(TestCase): +class SecurityReportCspTest(TestCase): @fixture def path(self): - path = reverse('sentry-api-csp-report', kwargs={'project_id': self.project.id}) + path = reverse('sentry-api-security-report', kwargs={'project_id': self.project.id}) return path + '?sentry_key=%s' % self.projectkey.public_key def test_get_response(self): @@ -44,7 +44,7 @@ def test_bad_origin(self, get_origins): resp = self.client.post( self.path, content_type='application/csp-report', - data='{"csp-report":{"document-uri":"http://lolnope.com"}}', + data='{"csp-report":{"document-uri":"http://lolnope.com","effective-directive":"img-src","violated-directive":"img-src","source-file":"test.html"}}', HTTP_USER_AGENT='awesome', ) assert resp.status_code == 403, resp.content @@ -56,10 +56,10 @@ def test_bad_origin(self, get_origins): data='{"csp-report":{"document-uri":"about:blank"}}', HTTP_USER_AGENT='awesome', ) - assert resp.status_code == 403, resp.content + assert resp.status_code == 400, resp.content @mock.patch('sentry.web.api.is_valid_origin', mock.Mock(return_value=True)) - @mock.patch('sentry.web.api.CspReportView.process') + @mock.patch('sentry.web.api.SecurityReportView.process') def test_post_success(self, process): process.return_value = 'ok' resp = self._postCspWithHeader( @@ -67,11 +67,78 @@ def test_post_success(self, process): 'document-uri': 'http://example.com', 'source-file': 'http://example.com', 'effective-directive': 'style-src', + 'violated-directive': 'style-src', } ) assert resp.status_code == 201, resp.content +class SecurityReportExpectCTTest(TestCase): + @fixture + def path(self): + path = reverse('sentry-api-security-report', kwargs={'project_id': self.project.id}) + return path + '?sentry_key=%s' % self.projectkey.public_key + + @mock.patch('sentry.web.api.is_valid_origin', mock.Mock(return_value=True)) + @mock.patch('sentry.web.api.SecurityReportView.process') + def test_post_success(self, process): + process.return_value = 'ok' + resp = self.client.post( + self.path, + content_type='application/expect-ct-report+json', + data=json.dumps({ + "expect-ct-report": { + "date-time": "2014-04-06T13:00:50Z", + "hostname": "www.example.com", + "port": 443, + "effective-expiration-date": "2014-05-01T12:40:50Z", + "served-certificate-chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + "validated-certificate-chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + "scts": [ + { + "version": 1, + "status": "invalid", + "source": "embedded", + "serialized_sct": "ABCD==" + }, + ], + } + }), + HTTP_USER_AGENT='awesome', + ) + assert resp.status_code == 201, resp.content + + +class SecurityReportExpectStapleTest(TestCase): + @fixture + def path(self): + path = reverse('sentry-api-security-report', kwargs={'project_id': self.project.id}) + return path + '?sentry_key=%s' % self.projectkey.public_key + + @mock.patch('sentry.web.api.is_valid_origin', mock.Mock(return_value=True)) + @mock.patch('sentry.web.api.SecurityReportView.process') + def test_post_success(self, process): + process.return_value = 'ok' + resp = self.client.post( + self.path, + content_type='application/expect-staple-report', + data=json.dumps({ + "expect-staple-report": { + "date-time": "2014-04-06T13:00:50Z", + "hostname": "www.example.com", + "port": 443, + "response-status": "ERROR_RESPONSE", + "cert-status": "REVOKED", + "effective-expiration-date": "2014-05-01T12:40:50Z", + "served-certificate-chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + "validated-certificate-chain": ["-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"], + } + }), + HTTP_USER_AGENT='awesome', + ) + assert resp.status_code == 201, resp.content + + class StoreViewTest(TestCase): @fixture def path(self):