diff --git a/src/sentry/api/endpoints/chunk.py b/src/sentry/api/endpoints/chunk.py index c9e65c33cfd13d..ee5b719248d9d6 100644 --- a/src/sentry/api/endpoints/chunk.py +++ b/src/sentry/api/endpoints/chunk.py @@ -25,6 +25,11 @@ MAX_CONCURRENCY = settings.DEBUG and 1 or 8 HASH_ALGORITHM = 'sha1' +CHUNK_UPLOAD_ACCEPT = ( + 'debug_files', # DIF assemble + 'release_files', # Artifacts assemble +) + class GzipChunk(BytesIO): def __init__(self, file): @@ -61,6 +66,7 @@ def get(self, request, organization): 'concurrency': MAX_CONCURRENCY, 'hashAlgorithm': HASH_ALGORITHM, 'compression': ['gzip'], + 'accept': CHUNK_UPLOAD_ACCEPT, } ) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 52a3fd208f93f1..5d12357071e027 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -18,8 +18,9 @@ from sentry.api.paginator import OffsetPaginator from sentry.api.serializers import serialize from sentry.constants import KNOWN_DIF_FORMATS -from sentry.models import ChunkFileState, FileBlobOwner, ProjectDebugFile, \ - create_files_from_dif_zip, get_assemble_status, set_assemble_status +from sentry.models import FileBlobOwner, ProjectDebugFile, create_files_from_dif_zip +from sentry.tasks.assemble import get_assemble_status, set_assemble_status, \ + AssembleTask, ChunkFileState from sentry.utils import json try: @@ -252,7 +253,10 @@ def post(self, request, project): "name": {"type": "string"}, "chunks": { "type": "array", - "items": {"type": "string"} + "items": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + } } }, "additionalProperties": False @@ -273,7 +277,6 @@ def post(self, request, project): file_response = {} - from sentry.tasks.assemble import assemble_dif for checksum, file_to_assemble in six.iteritems(files): name = file_to_assemble.get('name', None) chunks = file_to_assemble.get('chunks', []) @@ -281,7 +284,7 @@ def post(self, request, project): # First, check the cached assemble status. During assembling, a # ProjectDebugFile will be created and we need to prevent a race # condition. - state, detail = get_assemble_status(project, checksum) + state, detail = get_assemble_status(AssembleTask.DIF, project.id, checksum) if state == ChunkFileState.OK: file_response[checksum] = { 'state': state, @@ -337,7 +340,10 @@ def post(self, request, project): # We don't have a state yet, this means we can now start # an assemble job in the background. - set_assemble_status(project, checksum, ChunkFileState.CREATED) + set_assemble_status(AssembleTask.DIF, project.id, checksum, + ChunkFileState.CREATED) + + from sentry.tasks.assemble import assemble_dif assemble_dif.apply_async( kwargs={ 'project_id': project.id, diff --git a/src/sentry/api/endpoints/organization_release_assemble.py b/src/sentry/api/endpoints/organization_release_assemble.py new file mode 100644 index 00000000000000..a53ea0b034548b --- /dev/null +++ b/src/sentry/api/endpoints/organization_release_assemble.py @@ -0,0 +1,105 @@ +from __future__ import absolute_import + +import jsonschema +from rest_framework.response import Response + +from sentry.api.bases.organization import OrganizationReleasesBaseEndpoint +from sentry.api.exceptions import ResourceDoesNotExist +from sentry.models import Release +from sentry.tasks.assemble import get_assemble_status, set_assemble_status, \ + AssembleTask, ChunkFileState +from sentry.utils import json + + +class OrganizationReleaseAssembleEndpoint(OrganizationReleasesBaseEndpoint): + def post(self, request, organization, version): + """ + Handle an artifact bundle and merge it into the release + ``````````````````````````````````````````````````````` + + :auth: required + """ + + try: + release = Release.objects.get( + organization_id=organization.id, + version=version, + ) + except Release.DoesNotExist: + raise ResourceDoesNotExist + + if not self.has_release_permission(request, organization, release): + raise ResourceDoesNotExist + + schema = { + "type": "object", + "properties": { + "checksum": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + }, + "chunks": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + } + } + }, + "required": ["checksum", "chunks"], + "additionalProperties": False, + } + + try: + data = json.loads(request.body) + jsonschema.validate(data, schema) + except jsonschema.ValidationError as e: + return Response({'error': str(e).splitlines()[0]}, + status=400) + except BaseException as e: + return Response({'error': 'Invalid json body'}, + status=400) + + checksum = data.get('checksum', None) + chunks = data.get('chunks', []) + + state, detail = get_assemble_status(AssembleTask.ARTIFACTS, organization.id, checksum) + if state == ChunkFileState.OK: + return Response({ + 'state': state, + 'detail': None, + 'missingChunks': [], + }, status=200) + elif state is not None: + return Response({ + 'state': state, + 'detail': detail, + 'missingChunks': [], + }) + + # There is neither a known file nor a cached state, so we will + # have to create a new file. Assure that there are checksums. + # If not, we assume this is a poll and report NOT_FOUND + if not chunks: + return Response({ + 'state': ChunkFileState.NOT_FOUND, + 'missingChunks': [], + }, status=200) + + set_assemble_status(AssembleTask.ARTIFACTS, organization.id, checksum, + ChunkFileState.CREATED) + + from sentry.tasks.assemble import assemble_artifacts + assemble_artifacts.apply_async( + kwargs={ + 'org_id': organization.id, + 'version': version, + 'checksum': checksum, + 'chunks': chunks, + } + ) + + return Response({ + 'state': ChunkFileState.CREATED, + 'missingChunks': [], + }, status=200) diff --git a/src/sentry/api/urls.py b/src/sentry/api/urls.py index d7858fb24e502a..af583ba717880a 100644 --- a/src/sentry/api/urls.py +++ b/src/sentry/api/urls.py @@ -105,6 +105,7 @@ from .endpoints.organization_recent_searches import OrganizationRecentSearchesEndpoint from .endpoints.organization_releases import OrganizationReleasesEndpoint from .endpoints.organization_release_details import OrganizationReleaseDetailsEndpoint +from .endpoints.organization_release_assemble import OrganizationReleaseAssembleEndpoint from .endpoints.organization_release_files import OrganizationReleaseFilesEndpoint from .endpoints.organization_release_file_details import OrganizationReleaseFileDetailsEndpoint from .endpoints.organization_release_commits import OrganizationReleaseCommitsEndpoint @@ -743,6 +744,11 @@ OrganizationReleaseDetailsEndpoint.as_view(), name='sentry-api-0-organization-release-details' ), + url( + r'^organizations/(?P[^\/]+)/releases/(?P[^/]+)/assemble/$', + OrganizationReleaseAssembleEndpoint.as_view(), + name='sentry-api-0-organization-release-assemble' + ), url( r'^organizations/(?P[^\/]+)/releases/(?P[^/]+)/files/$', OrganizationReleaseFilesEndpoint.as_view(), diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index ee97194620c7bf..c6d352fb52f7db 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -24,7 +24,6 @@ from symbolic import Archive, SymbolicError, ObjectErrorUnsupportedObject from sentry import options -from sentry.cache import default_cache from sentry.constants import KNOWN_DIF_FORMATS from sentry.db.models import FlexibleForeignKey, Model, sane_repr, BaseManager, JSONField from sentry.models.file import File @@ -47,37 +46,6 @@ _proguard_file_re = re.compile(r'/proguard/(?:mapping-)?(.*?)\.txt$') -def _get_idempotency_id(project, checksum): - """For some operations an idempotency ID is needed.""" - return hashlib.sha1(b'%s|%s|project.dsym' % ( - str(project.id).encode('ascii'), - checksum.encode('ascii'), - )).hexdigest() - - -def get_assemble_status(project, checksum): - """For a given file it checks what the current status of the assembling is. - Returns a tuple in the form ``(status, details)`` where details is either - `None` or a string identifying an error condition or notice. - """ - cache_key = 'assemble-status:%s' % _get_idempotency_id( - project, checksum) - rv = default_cache.get(cache_key) - if rv is None: - return None, None - return tuple(rv) - - -def set_assemble_status(project, checksum, state, detail=None): - cache_key = 'assemble-status:%s' % _get_idempotency_id( - project, checksum) - - # NB: Also cache successfully created debug files to avoid races between - # multiple DIFs with the same identifier. On the downside, this blocks - # re-uploads for 10 minutes. - default_cache.set(cache_key, (state, detail), 600) - - class BadDif(Exception): pass diff --git a/src/sentry/models/file.py b/src/sentry/models/file.py index f85aa28a1ed14c..f5d8cc067a25f0 100644 --- a/src/sentry/models/file.py +++ b/src/sentry/models/file.py @@ -57,19 +57,6 @@ class nooplogger(object): exception = staticmethod(lambda *a, **kw: None) -def enum(**named_values): - return type('Enum', (), named_values) - - -ChunkFileState = enum( - OK='ok', # File in database - NOT_FOUND='not_found', # File not found in database - CREATED='created', # File was created in the request and send to the worker for assembling - ASSEMBLING='assembling', # File still being processed by worker - ERROR='error' # Error happened during assembling -) - - def _get_size_and_checksum(fileobj, logger=nooplogger): logger.info('_get_size_and_checksum.start') size = 0 diff --git a/src/sentry/tasks/assemble.py b/src/sentry/tasks/assemble.py index 111cfa4864be3f..f6c8f2940bf564 100644 --- a/src/sentry/tasks/assemble.py +++ b/src/sentry/tasks/assemble.py @@ -1,29 +1,100 @@ from __future__ import absolute_import, print_function +from os import path + +import hashlib import logging +import six + +from django.db import IntegrityError, transaction from sentry.api.serializers import serialize +from sentry.cache import default_cache from sentry.tasks.base import instrumented_task +from sentry.utils import json from sentry.utils.files import get_max_file_size from sentry.utils.sdk import configure_scope logger = logging.getLogger(__name__) +def enum(**named_values): + """Creates an enum type.""" + return type('Enum', (), named_values) + + +ChunkFileState = enum( + OK='ok', # File in database + NOT_FOUND='not_found', # File not found in database + CREATED='created', # File was created in the request and send to the worker for assembling + ASSEMBLING='assembling', # File still being processed by worker + ERROR='error' # Error happened during assembling +) + + +AssembleTask = enum( + DIF='project.dsym', # Debug file upload + ARTIFACTS='organization.artifacts', # Release file upload +) + + +def _get_cache_key(task, scope, checksum): + """Computes the cache key for assemble status. + + ``task`` must be one of the ``AssembleTask`` values. The scope can be the + identifier of any model, such as the organization or project that this task + is performed under. + + ``checksum`` should be the SHA1 hash of the main file that is being + assembled. + """ + return 'assemble-status:%s' % hashlib.sha1(b'%s|%s|%s' % ( + str(scope).encode('ascii'), + checksum.encode('ascii'), + task, + )).hexdigest() + + +def get_assemble_status(task, scope, checksum): + """ + Checks the current status of an assembling task. + + Returns a tuple in the form ``(status, details)``, where ``status`` is the + ChunkFileState, and ``details`` is either None or a string containing a + notice or error message. + """ + cache_key = _get_cache_key(task, scope, checksum) + rv = default_cache.get(cache_key) + if rv is None: + return None, None + return tuple(rv) + + +def set_assemble_status(task, scope, checksum, state, detail=None): + """ + Updates the status of an assembling task. It is cached for 10 minutes. + """ + cache_key = _get_cache_key(task, scope, checksum) + default_cache.set(cache_key, (state, detail), 600) + + @instrumented_task(name='sentry.tasks.assemble.assemble_dif', queue='assemble') def assemble_dif(project_id, name, checksum, chunks, **kwargs): - from sentry.models import ChunkFileState, debugfile, Project, \ - set_assemble_status, BadDif + """ + Assembles uploaded chunks into a ``ProjectDebugFile``. + """ + + from sentry.models import debugfile, Project, BadDif from sentry.reprocessing import bump_reprocessing_revision with configure_scope() as scope: scope.set_tag("project", project_id) project = Project.objects.filter(id=project_id).get() - set_assemble_status(project, checksum, ChunkFileState.ASSEMBLING) + set_assemble_status(AssembleTask.DIF, project.id, checksum, ChunkFileState.ASSEMBLING) - # Assemble the chunks into files - rv = assemble_file(project, name, checksum, chunks, + # Assemble the chunks into a temporary file + rv = assemble_file(AssembleTask.DIF, project.id, name, checksum, chunks, file_type='project.dif') # If not file has been created this means that the file failed to @@ -40,19 +111,17 @@ def assemble_dif(project_id, name, checksum, chunks, **kwargs): try: result = debugfile.detect_dif_from_path(temp_file.name, name=name) except BadDif as e: - set_assemble_status(project, checksum, ChunkFileState.ERROR, - detail=e.args[0]) + set_assemble_status(AssembleTask.DIF, project.id, checksum, + ChunkFileState.ERROR, detail=e.args[0]) return if len(result) != 1: - set_assemble_status(project, checksum, ChunkFileState.ERROR, - detail='Contained wrong number of ' - 'architectures (expected one, got %s)' - % len(result)) + detail = 'Object contains %s architectures (1 expected)' % len(result) + set_assemble_status(AssembleTask.DIF, project.id, checksum, + ChunkFileState.ERROR, detail=detail) return dif, created = debugfile.create_dif_from_id(project, result[0], file=file) - indicate_success = True delete_file = False if created: @@ -61,19 +130,163 @@ def assemble_dif(project_id, name, checksum, chunks, **kwargs): # created, someone else has created it and will bump the # revision instead. bump_reprocessing_revision(project) - - if indicate_success: - set_assemble_status(project, checksum, ChunkFileState.OK, - detail=serialize(dif)) + except BaseException: + set_assemble_status(AssembleTask.DIF, project.id, checksum, ChunkFileState.ERROR, + detail='internal server error') + logger.error('failed to assemble dif', exc_info=True) + else: + set_assemble_status(AssembleTask.DIF, project.id, checksum, ChunkFileState.OK, + detail=serialize(dif)) finally: if delete_file: file.delete() -def assemble_file(project, name, checksum, chunks, file_type): - '''This assembles multiple chunks into on File.''' - from sentry.models import File, ChunkFileState, AssembleChecksumMismatch, \ - FileBlob, set_assemble_status +class AssembleArtifactsError(Exception): + pass + + +@instrumented_task(name='sentry.tasks.assemble.assemble_artifacts', queue='assemble') +def assemble_artifacts(org_id, version, checksum, chunks, **kwargs): + """ + Creates release files from an uploaded artifact bundle. + """ + + import shutil + import tempfile + from sentry.utils.zip import safe_extract_zip + from sentry.models import File, Organization, Release, ReleaseFile + + with configure_scope() as scope: + scope.set_tag("organization", org_id) + + organization = Organization.objects.filter(id=org_id).get() + set_assemble_status(AssembleTask.ARTIFACTS, org_id, checksum, ChunkFileState.ASSEMBLING) + + # Assemble the chunks into a temporary file + rv = assemble_file(AssembleTask.ARTIFACTS, organization, 'release-artifacts.zip', + checksum, chunks, file_type='release.bundle') + + # If not file has been created this means that the file failed to + # assemble because of bad input data. Return. + if rv is None: + return + + bundle, temp_file = rv + scratchpad = tempfile.mkdtemp() + + # Initially, always delete the bundle file. Later on, we can start to store + # the artifact bundle as a release file. + delete_bundle = True + + try: + try: + safe_extract_zip(temp_file, scratchpad, strip_toplevel=False) + except BaseException: + raise AssembleArtifactsError('failed to extract bundle') + + try: + manifest_path = path.join(scratchpad, 'manifest.json') + with open(manifest_path, 'rb') as manifest: + manifest = json.loads(manifest.read()) + except BaseException: + raise AssembleArtifactsError('failed to open release manifest') + + org_slug = manifest.get('org') + if organization.slug != org_slug: + raise AssembleArtifactsError('organization does not match uploaded bundle') + + release_name = manifest.get('release') + if release_name != version: + raise AssembleArtifactsError('release does not match uploaded bundle') + + try: + release = Release.objects.get( + organization_id=organization.id, + version=release_name + ) + except Release.DoesNotExist: + raise AssembleArtifactsError('release does not exist') + + dist_name = manifest.get('dist') + dist = None + if dist_name: + dist = release.add_dist(dist_name) + + artifacts = manifest.get('files', {}) + for rel_path, artifact in six.iteritems(artifacts): + artifact_url = artifact.get('url', rel_path) + artifact_basename = artifact_url.rsplit('/', 1)[-1] + + file = File.objects.create( + name=artifact_basename, + type='release.file', + headers=artifact.get('headers', {}) + ) + + full_path = path.join(scratchpad, rel_path) + with open(full_path, 'rb') as fp: + file.putfile(fp, logger=logger) + + kwargs = { + 'organization_id': organization.id, + 'release': release, + 'name': artifact_url, + 'dist': dist, + } + + # Release files must have unique names within their release + # and dist. If a matching file already exists, replace its + # file with the new one; otherwise create it. + try: + release_file = ReleaseFile.objects.get(**kwargs) + except ReleaseFile.DoesNotExist: + try: + with transaction.atomic(): + ReleaseFile.objects.create(file=file, **kwargs) + except IntegrityError: + # NB: This indicates a race, where another assemble task or + # file upload job has just created a conflicting file. Since + # we're upserting here anyway, yield to the faster actor and + # do not try again. + file.delete() + else: + old_file = release_file.file + release_file.update(file=file) + old_file.delete() + + except AssembleArtifactsError as e: + set_assemble_status(AssembleTask.ARTIFACTS, org_id, checksum, + ChunkFileState.ERROR, detail=e.message) + except BaseException: + logger.error('failed to assemble release bundle', exc_info=True) + set_assemble_status(AssembleTask.ARTIFACTS, org_id, checksum, + ChunkFileState.ERROR, detail='internal server error') + else: + set_assemble_status(AssembleTask.ARTIFACTS, org_id, checksum, + ChunkFileState.OK) + finally: + shutil.rmtree(scratchpad) + if delete_bundle: + bundle.delete() + + +def assemble_file(task, org_or_project, name, checksum, chunks, file_type): + """ + Verifies and assembles a file model from chunks. + + This downloads all chunks from blob store to verify their integrity and + associates them with a created file model. Additionally, it assembles the + full file in a temporary location and verifies the complete content hash. + + Returns a tuple ``(File, TempFile)`` on success, or ``None`` on error. + """ + from sentry.models import File, AssembleChecksumMismatch, FileBlob, Project + + if isinstance(org_or_project, Project): + organization = org_or_project.organization + else: + organization = org_or_project # Load all FileBlobs from db since we can be sure here we already own all # chunks need to build the file @@ -84,8 +297,8 @@ def assemble_file(project, name, checksum, chunks, file_type): # Reject all files that exceed the maximum allowed size for this # organization. This value cannot be file_size = sum(x[2] for x in file_blobs) - if file_size > get_max_file_size(project.organization): - set_assemble_status(project, checksum, ChunkFileState.ERROR, + if file_size > get_max_file_size(organization): + set_assemble_status(task, org_or_project.id, checksum, ChunkFileState.ERROR, detail='File exceeds maximum size') return @@ -100,7 +313,7 @@ def assemble_file(project, name, checksum, chunks, file_type): # Sanity check. In case not all blobs exist at this point we have a # race condition. if set(x[1] for x in file_blobs) != set(chunks): - set_assemble_status(project, checksum, ChunkFileState.ERROR, + set_assemble_status(task, org_or_project.id, checksum, ChunkFileState.ERROR, detail='Not all chunks available for assembling') return @@ -113,7 +326,7 @@ def assemble_file(project, name, checksum, chunks, file_type): temp_file = file.assemble_from_file_blob_ids(file_blob_ids, checksum) except AssembleChecksumMismatch: file.delete() - set_assemble_status(project, checksum, ChunkFileState.ERROR, + set_assemble_status(task, org_or_project.id, checksum, ChunkFileState.ERROR, detail='Reported checksum mismatch') else: file.save() diff --git a/src/sentry/testutils/factories.py b/src/sentry/testutils/factories.py index 866eed37e47fb4..1e805bb6f74c00 100644 --- a/src/sentry/testutils/factories.py +++ b/src/sentry/testutils/factories.py @@ -5,7 +5,7 @@ from django.utils.importlib import import_module import copy -import json +import io import os import petname import random @@ -35,11 +35,24 @@ UserReport, PlatformExternalIssue, ) from sentry.models.integrationfeature import Feature, IntegrationFeature +from sentry.utils import json from sentry.utils.canonical import CanonicalKeyDict loremipsum = Generator() +def get_fixture_path(name): + return os.path.join( + os.path.dirname(__file__), # src/sentry/testutils/ + os.pardir, # src/sentry/ + os.pardir, # src/ + os.pardir, + 'tests', + 'fixtures', + name + ) + + def make_sentence(words=None): if words is None: words = int(random.weibullvariate(8, 3)) @@ -173,6 +186,15 @@ def make_word(words=None): } +def _patch_artifact_manifest(path, org, release, project=None): + manifest = json.loads(open(path, 'rb').read()) + manifest['org'] = org + manifest['release'] = release + if project: + manifest['project'] = project + return json.dumps(manifest) + + # TODO(dcramer): consider moving to something more scaleable like factoryboy class Factories(object): @staticmethod @@ -312,6 +334,25 @@ def create_release(project, user=None, version=None, date_added=None): return release + @staticmethod + def create_artifact_bundle(org, release, project=None): + import zipfile + + bundle = io.BytesIO() + bundle_dir = get_fixture_path('artifact_bundle') + with zipfile.ZipFile(bundle, 'w', zipfile.ZIP_DEFLATED) as zipfile: + for path, _, files in os.walk(bundle_dir): + for filename in files: + fullpath = os.path.join(path, filename) + relpath = os.path.relpath(fullpath, bundle_dir) + if filename == 'manifest.json': + manifest = _patch_artifact_manifest(fullpath, org, release, project) + zipfile.writestr(relpath, manifest) + else: + zipfile.write(fullpath, relpath) + + return bundle.getvalue() + @staticmethod def create_repo(project, name=None): repo = Repository.objects.create( diff --git a/src/sentry/testutils/fixtures.py b/src/sentry/testutils/fixtures.py index 6ca347510a4fe7..e5529060ca9ec6 100644 --- a/src/sentry/testutils/fixtures.py +++ b/src/sentry/testutils/fixtures.py @@ -124,6 +124,13 @@ def create_release(self, project=None, user=None, *args, **kwargs): project = self.project return Factories.create_release(project=project, user=user, *args, **kwargs) + def create_artifact_bundle(self, org=None, release=None, *args, **kwargs): + if org is None: + org = self.organization.slug + if release is None: + release = self.release.version + return Factories.create_artifact_bundle(org, release, *args, **kwargs) + def create_repo(self, project=None, *args, **kwargs): if project is None: project = self.project diff --git a/tests/fixtures/artifact_bundle/files/_/_/index.js b/tests/fixtures/artifact_bundle/files/_/_/index.js new file mode 100644 index 00000000000000..a288b3d488e855 --- /dev/null +++ b/tests/fixtures/artifact_bundle/files/_/_/index.js @@ -0,0 +1,126 @@ +Object.defineProperty(exports, "__esModule", { value: true }); +var tslib_1 = require("tslib"); +var hub_1 = require("@sentry/hub"); +/** + * This calls a function on the current hub. + * @param method function to call on hub. + * @param args to pass to function. + */ +function callOnHub(method) { + var args = []; + for (var _i = 1; _i < arguments.length; _i++) { + args[_i - 1] = arguments[_i]; + } + var hub = hub_1.getCurrentHub(); + if (hub && hub[method]) { + // tslint:disable-next-line:no-unsafe-any + return hub[method].apply(hub, tslib_1.__spread(args)); + } + throw new Error("No hub defined or " + method + " was not found on the hub, please open a bug report."); +} +/** + * Captures an exception event and sends it to Sentry. + * + * @param exception An exception-like object. + * @returns The generated eventId. + */ +function captureException(exception) { + var syntheticException; + try { + throw new Error('Sentry syntheticException'); + } + catch (exception) { + syntheticException = exception; + } + return callOnHub('captureException', exception, { + originalException: exception, + syntheticException: syntheticException, + }); +} +exports.captureException = captureException; +/** + * Captures a message event and sends it to Sentry. + * + * @param message The message to send to Sentry. + * @param level Define the level of the message. + * @returns The generated eventId. + */ +function captureMessage(message, level) { + var syntheticException; + try { + throw new Error(message); + } + catch (exception) { + syntheticException = exception; + } + return callOnHub('captureMessage', message, level, { + originalException: message, + syntheticException: syntheticException, + }); +} +exports.captureMessage = captureMessage; +/** + * Captures a manually created event and sends it to Sentry. + * + * @param event The event to send to Sentry. + * @returns The generated eventId. + */ +function captureEvent(event) { + return callOnHub('captureEvent', event); +} +exports.captureEvent = captureEvent; +/** + * Records a new breadcrumb which will be attached to future events. + * + * Breadcrumbs will be added to subsequent events to provide more context on + * user's actions prior to an error or crash. + * + * @param breadcrumb The breadcrumb to record. + */ +function addBreadcrumb(breadcrumb) { + callOnHub('addBreadcrumb', breadcrumb); +} +exports.addBreadcrumb = addBreadcrumb; +/** + * Callback to set context information onto the scope. + * @param callback Callback function that receives Scope. + */ +function configureScope(callback) { + callOnHub('configureScope', callback); +} +exports.configureScope = configureScope; +/** + * Creates a new scope with and executes the given operation within. + * The scope is automatically removed once the operation + * finishes or throws. + * + * This is essentially a convenience function for: + * + * pushScope(); + * callback(); + * popScope(); + * + * @param callback that will be enclosed into push/popScope. + */ +function withScope(callback) { + callOnHub('withScope', callback); +} +exports.withScope = withScope; +/** + * Calls a function on the latest client. Use this with caution, it's meant as + * in "internal" helper so we don't need to expose every possible function in + * the shim. It is not guaranteed that the client actually implements the + * function. + * + * @param method The method to call on the client/client. + * @param args Arguments to pass to the client/fontend. + */ +function _callOnClient(method) { + var args = []; + for (var _i = 1; _i < arguments.length; _i++) { + args[_i - 1] = arguments[_i]; + } + callOnHub.apply(void 0, tslib_1.__spread(['_invokeClient', method], args)); +} +exports._callOnClient = _callOnClient; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/tests/fixtures/artifact_bundle/files/_/_/index.js.map b/tests/fixtures/artifact_bundle/files/_/_/index.js.map new file mode 100644 index 00000000000000..f12f94a9d9e1e5 --- /dev/null +++ b/tests/fixtures/artifact_bundle/files/_/_/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,mCAAwD;AAGxD;;;;GAIG;AACH,SAAS,SAAS,CAAI,MAAc;IAAE,cAAc;SAAd,UAAc,EAAd,qBAAc,EAAd,IAAc;QAAd,6BAAc;;IAClD,IAAM,GAAG,GAAG,mBAAa,EAAE,CAAC;IAC5B,IAAI,GAAG,IAAI,GAAG,CAAC,MAAmB,CAAC,EAAE;QACnC,yCAAyC;QACzC,OAAQ,GAAG,CAAC,MAAmB,CAAC,OAAxB,GAAG,mBAAiC,IAAI,GAAE;KACnD;IACD,MAAM,IAAI,KAAK,CAAC,uBAAqB,MAAM,yDAAsD,CAAC,CAAC;AACrG,CAAC;AAED;;;;;GAKG;AACH,SAAgB,gBAAgB,CAAC,SAAc;IAC7C,IAAI,kBAAyB,CAAC;IAC9B,IAAI;QACF,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;KAC9C;IAAC,OAAO,SAAS,EAAE;QAClB,kBAAkB,GAAG,SAAkB,CAAC;KACzC;IACD,OAAO,SAAS,CAAC,kBAAkB,EAAE,SAAS,EAAE;QAC9C,iBAAiB,EAAE,SAAS;QAC5B,kBAAkB,oBAAA;KACnB,CAAC,CAAC;AACL,CAAC;AAXD,4CAWC;AAED;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,OAAe,EAAE,KAAgB;IAC9D,IAAI,kBAAyB,CAAC;IAC9B,IAAI;QACF,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;KAC1B;IAAC,OAAO,SAAS,EAAE;QAClB,kBAAkB,GAAG,SAAkB,CAAC;KACzC;IACD,OAAO,SAAS,CAAC,gBAAgB,EAAE,OAAO,EAAE,KAAK,EAAE;QACjD,iBAAiB,EAAE,OAAO;QAC1B,kBAAkB,oBAAA;KACnB,CAAC,CAAC;AACL,CAAC;AAXD,wCAWC;AAED;;;;;GAKG;AACH,SAAgB,YAAY,CAAC,KAAY;IACvC,OAAO,SAAS,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;AAC1C,CAAC;AAFD,oCAEC;AAED;;;;;;;GAOG;AACH,SAAgB,aAAa,CAAC,UAAsB;IAClD,SAAS,CAAO,eAAe,EAAE,UAAU,CAAC,CAAC;AAC/C,CAAC;AAFD,sCAEC;AAED;;;GAGG;AACH,SAAgB,cAAc,CAAC,QAAgC;IAC7D,SAAS,CAAO,gBAAgB,EAAE,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAFD,wCAEC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,SAAS,CAAC,QAAgC;IACxD,SAAS,CAAO,WAAW,EAAE,QAAQ,CAAC,CAAC;AACzC,CAAC;AAFD,8BAEC;AAED;;;;;;;;GAQG;AACH,SAAgB,aAAa,CAAC,MAAc;IAAE,cAAc;SAAd,UAAc,EAAd,qBAAc,EAAd,IAAc;QAAd,6BAAc;;IAC1D,SAAS,iCAAO,eAAe,EAAE,MAAM,GAAK,IAAI,GAAE;AACpD,CAAC;AAFD,sCAEC","sourcesContent":["import { getCurrentHub, Hub, Scope } from '@sentry/hub';\nimport { Breadcrumb, Event, Severity } from '@sentry/types';\n\n/**\n * This calls a function on the current hub.\n * @param method function to call on hub.\n * @param args to pass to function.\n */\nfunction callOnHub(method: string, ...args: any[]): T {\n const hub = getCurrentHub();\n if (hub && hub[method as keyof Hub]) {\n // tslint:disable-next-line:no-unsafe-any\n return (hub[method as keyof Hub] as any)(...args);\n }\n throw new Error(`No hub defined or ${method} was not found on the hub, please open a bug report.`);\n}\n\n/**\n * Captures an exception event and sends it to Sentry.\n *\n * @param exception An exception-like object.\n * @returns The generated eventId.\n */\nexport function captureException(exception: any): string {\n let syntheticException: Error;\n try {\n throw new Error('Sentry syntheticException');\n } catch (exception) {\n syntheticException = exception as Error;\n }\n return callOnHub('captureException', exception, {\n originalException: exception,\n syntheticException,\n });\n}\n\n/**\n * Captures a message event and sends it to Sentry.\n *\n * @param message The message to send to Sentry.\n * @param level Define the level of the message.\n * @returns The generated eventId.\n */\nexport function captureMessage(message: string, level?: Severity): string {\n let syntheticException: Error;\n try {\n throw new Error(message);\n } catch (exception) {\n syntheticException = exception as Error;\n }\n return callOnHub('captureMessage', message, level, {\n originalException: message,\n syntheticException,\n });\n}\n\n/**\n * Captures a manually created event and sends it to Sentry.\n *\n * @param event The event to send to Sentry.\n * @returns The generated eventId.\n */\nexport function captureEvent(event: Event): string {\n return callOnHub('captureEvent', event);\n}\n\n/**\n * Records a new breadcrumb which will be attached to future events.\n *\n * Breadcrumbs will be added to subsequent events to provide more context on\n * user's actions prior to an error or crash.\n *\n * @param breadcrumb The breadcrumb to record.\n */\nexport function addBreadcrumb(breadcrumb: Breadcrumb): void {\n callOnHub('addBreadcrumb', breadcrumb);\n}\n\n/**\n * Callback to set context information onto the scope.\n * @param callback Callback function that receives Scope.\n */\nexport function configureScope(callback: (scope: Scope) => void): void {\n callOnHub('configureScope', callback);\n}\n\n/**\n * Creates a new scope with and executes the given operation within.\n * The scope is automatically removed once the operation\n * finishes or throws.\n *\n * This is essentially a convenience function for:\n *\n * pushScope();\n * callback();\n * popScope();\n *\n * @param callback that will be enclosed into push/popScope.\n */\nexport function withScope(callback: (scope: Scope) => void): void {\n callOnHub('withScope', callback);\n}\n\n/**\n * Calls a function on the latest client. Use this with caution, it's meant as\n * in \"internal\" helper so we don't need to expose every possible function in\n * the shim. It is not guaranteed that the client actually implements the\n * function.\n *\n * @param method The method to call on the client/client.\n * @param args Arguments to pass to the client/fontend.\n */\nexport function _callOnClient(method: string, ...args: any[]): void {\n callOnHub('_invokeClient', method, ...args);\n}\n"]} \ No newline at end of file diff --git a/tests/fixtures/artifact_bundle/manifest.json b/tests/fixtures/artifact_bundle/manifest.json new file mode 100644 index 00000000000000..6b874573426765 --- /dev/null +++ b/tests/fixtures/artifact_bundle/manifest.json @@ -0,0 +1,17 @@ +{ + "org": "__org__", + "release": "__release__", + "files": { + "files/_/_/index.js.map": { + "url": "~/index.js.map", + "type": "source_map" + }, + "files/_/_/index.js": { + "url": "~/index.js", + "type": "source", + "headers": { + "Sourcemap": "index.js.map" + } + } + } +} diff --git a/tests/sentry/api/endpoints/test_dif_assemble.py b/tests/sentry/api/endpoints/test_dif_assemble.py index 20ce0dcf09b412..2ed33d8e76d2b4 100644 --- a/tests/sentry/api/endpoints/test_dif_assemble.py +++ b/tests/sentry/api/endpoints/test_dif_assemble.py @@ -7,10 +7,10 @@ from django.core.files.base import ContentFile from sentry.models import ApiToken, FileBlob, File, FileBlobIndex, FileBlobOwner -from sentry.models.file import ChunkFileState -from sentry.models.debugfile import get_assemble_status, set_assemble_status, ProjectDebugFile +from sentry.models.debugfile import ProjectDebugFile from sentry.testutils import APITestCase -from sentry.tasks.assemble import assemble_dif, assemble_file +from sentry.tasks.assemble import assemble_dif, assemble_file, get_assemble_status, \ + set_assemble_status, AssembleTask, ChunkFileState class DifAssembleEndpoint(APITestCase): @@ -141,7 +141,7 @@ def test_assemble_check(self): debug_id='df449af8-0dcd-4320-9943-ec192134d593', code_id='DF449AF80DCD43209943EC192134D593', ) - set_assemble_status(self.project, checksum, None) + set_assemble_status(AssembleTask.DIF, self.project.id, checksum, None) # Request now tells us that everything is alright response = self.client.post( @@ -255,8 +255,10 @@ def test_assemble(self, mock_assemble_dif): } ) - file = assemble_file(self.project, 'test', total_checksum, chunks, 'project.dif')[0] - assert get_assemble_status(self.project, total_checksum)[0] != ChunkFileState.ERROR + file = assemble_file(AssembleTask.DIF, self.project, 'test', + total_checksum, chunks, 'project.dif')[0] + status, _ = get_assemble_status(AssembleTask.DIF, self.project.id, total_checksum) + assert status != ChunkFileState.ERROR assert file.checksum == total_checksum file_blob_index = FileBlobIndex.objects.all() diff --git a/tests/sentry/api/endpoints/test_organization_release_assemble.py b/tests/sentry/api/endpoints/test_organization_release_assemble.py new file mode 100644 index 00000000000000..d1ad5d4acab6cb --- /dev/null +++ b/tests/sentry/api/endpoints/test_organization_release_assemble.py @@ -0,0 +1,147 @@ +from __future__ import absolute_import + +from mock import patch +from hashlib import sha1 + +from django.core.urlresolvers import reverse +from django.core.files.base import ContentFile + +from sentry.models import ApiToken, FileBlob, FileBlobOwner +from sentry.testutils import APITestCase +from sentry.tasks.assemble import assemble_artifacts, ChunkFileState + + +class OrganizationReleaseAssembleTest(APITestCase): + def setUp(self): + self.organization = self.create_organization(owner=self.user) + self.token = ApiToken.objects.create( + user=self.user, + scope_list=['project:write'], + ) + self.team = self.create_team(organization=self.organization) + self.release = self.create_release(version='my-unique-release.1') + self.url = reverse( + 'sentry-api-0-organization-release-assemble', + args=[self.organization.slug, self.release.version], + ) + + def test_assemble_json_schema(self): + response = self.client.post( + self.url, + data={ + 'lol': 'test' + }, + HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) + ) + assert response.status_code == 400, response.content + + checksum = sha1('1').hexdigest() + response = self.client.post( + self.url, + data={ + 'checksum': 'invalid' + }, + HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) + ) + assert response.status_code == 400, response.content + + response = self.client.post( + self.url, + data={ + 'checksum': checksum, + }, + HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) + ) + assert response.status_code == 400, response.content + + response = self.client.post( + self.url, + data={ + 'checksum': checksum, + 'chunks': [], + }, + HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) + ) + assert response.status_code == 200, response.content + assert response.data['state'] == ChunkFileState.NOT_FOUND + + @patch('sentry.tasks.assemble.assemble_artifacts') + def test_assemble(self, mock_assemble_artifacts): + bundle_file = self.create_artifact_bundle() + total_checksum = sha1(bundle_file).hexdigest() + + blob1 = FileBlob.from_file(ContentFile(bundle_file)) + FileBlobOwner.objects.get_or_create( + organization=self.organization, + blob=blob1 + ) + + response = self.client.post( + self.url, + data={ + 'checksum': total_checksum, + 'chunks': [blob1.checksum], + }, + HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) + ) + + assert response.status_code == 200, response.content + assert response.data['state'] == ChunkFileState.CREATED + assert set(response.data['missingChunks']) == set([]) + + mock_assemble_artifacts.apply_async.assert_called_once_with( + kwargs={ + 'org_id': self.organization.id, + 'version': self.release.version, + 'chunks': [blob1.checksum], + 'checksum': total_checksum, + } + ) + + def test_assemble_response(self): + bundle_file = self.create_artifact_bundle() + total_checksum = sha1(bundle_file).hexdigest() + blob1 = FileBlob.from_file(ContentFile(bundle_file)) + + assemble_artifacts( + org_id=self.organization.id, + version=self.release.version, + checksum=total_checksum, + chunks=[blob1.checksum], + ) + + response = self.client.post( + self.url, + data={ + 'checksum': total_checksum, + 'chunks': [blob1.checksum], + }, + HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) + ) + + assert response.status_code == 200, response.content + assert response.data['state'] == ChunkFileState.OK + + def test_dif_error_response(self): + bundle_file = b'invalid' + total_checksum = sha1(bundle_file).hexdigest() + blob1 = FileBlob.from_file(ContentFile(bundle_file)) + + assemble_artifacts( + org_id=self.organization.id, + version=self.release.version, + checksum=total_checksum, + chunks=[blob1.checksum], + ) + + response = self.client.post( + self.url, + data={ + 'checksum': total_checksum, + 'chunks': [blob1.checksum], + }, + HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) + ) + + assert response.status_code == 200, response.content + assert response.data['state'] == ChunkFileState.ERROR diff --git a/tests/sentry/tasks/test_assemble.py b/tests/sentry/tasks/test_assemble.py index a8218f2127d0f4..190ff4640bb773 100644 --- a/tests/sentry/tasks/test_assemble.py +++ b/tests/sentry/tasks/test_assemble.py @@ -9,13 +9,13 @@ from six.moves import xrange from sentry.testutils import TestCase -from sentry.tasks.assemble import assemble_dif, assemble_file -from sentry.models import FileBlob, FileBlobOwner -from sentry.models.file import ChunkFileState -from sentry.models.debugfile import get_assemble_status, ProjectDebugFile +from sentry.tasks.assemble import assemble_artifacts, assemble_dif, assemble_file, \ + get_assemble_status, AssembleTask, ChunkFileState +from sentry.models import FileBlob, FileBlobOwner, ReleaseFile +from sentry.models.debugfile import ProjectDebugFile -class AssembleTest(TestCase): +class BaseAssembleTest(TestCase): def setUp(self): self.organization = self.create_organization(owner=self.user) self.team = self.create_team(organization=self.organization) @@ -25,6 +25,8 @@ def setUp(self): organization=self.organization, name='foo') + +class AssembleDifTest(BaseAssembleTest): def test_wrong_dif(self): content1 = 'foo'.encode('utf-8') fileobj1 = ContentFile(content1) @@ -51,7 +53,8 @@ def test_wrong_dif(self): chunks=chunks, ) - assert get_assemble_status(self.project, total_checksum)[0] == ChunkFileState.ERROR + status, _ = get_assemble_status(AssembleTask.DIF, self.project.id, total_checksum) + assert status == ChunkFileState.ERROR def test_dif(self): sym_file = self.load_fixture('crash.sym') @@ -65,6 +68,9 @@ def test_dif(self): chunks=[blob1.checksum], ) + status, _ = get_assemble_status(AssembleTask.DIF, self.project.id, total_checksum) + assert status == ChunkFileState.OK + dif = ProjectDebugFile.objects.filter( project=self.project, file__checksum=total_checksum, @@ -94,9 +100,9 @@ def test_assemble_from_files(self): organization=self.organization ).get() - rv = assemble_file( - self.project, 'testfile', file_checksum.hexdigest(), - [x[1] for x in files], 'dummy.type') + rv = assemble_file(AssembleTask.DIF, + self.project, 'testfile', file_checksum.hexdigest(), + [x[1] for x in files], 'dummy.type') assert rv is not None f, tmp = rv @@ -109,7 +115,88 @@ def test_assemble_from_files(self): FileBlob.from_files(files, organization=self.organization) # assemble a second time - f = assemble_file( - self.project, 'testfile', file_checksum.hexdigest(), - [x[1] for x in files], 'dummy.type')[0] + f = assemble_file(AssembleTask.DIF, + self.project, 'testfile', file_checksum.hexdigest(), + [x[1] for x in files], 'dummy.type')[0] assert f.checksum == file_checksum.hexdigest() + + +class AssembleArtifactsTest(BaseAssembleTest): + def setUp(self): + super(AssembleArtifactsTest, self).setUp() + self.release = self.create_release(version='my-unique-release.1') + + def test_artifacts(self): + bundle_file = self.create_artifact_bundle() + blob1 = FileBlob.from_file(ContentFile(bundle_file)) + total_checksum = sha1(bundle_file).hexdigest() + + assemble_artifacts( + org_id=self.organization.id, + version=self.release.version, + checksum=total_checksum, + chunks=[blob1.checksum], + ) + + status, details = get_assemble_status(AssembleTask.ARTIFACTS, self.organization.id, + total_checksum) + assert status == ChunkFileState.OK + assert details is None + + release_file = ReleaseFile.objects.get( + organization=self.organization, + release=self.release, + name='~/index.js', + dist=None, + ) + + assert release_file + assert release_file.file.headers == {'Sourcemap': 'index.js.map'} + + def test_artifacts_invalid_org(self): + bundle_file = self.create_artifact_bundle(org='invalid') + blob1 = FileBlob.from_file(ContentFile(bundle_file)) + total_checksum = sha1(bundle_file).hexdigest() + + assemble_artifacts( + org_id=self.organization.id, + version=self.release.version, + checksum=total_checksum, + chunks=[blob1.checksum], + ) + + status, details = get_assemble_status(AssembleTask.ARTIFACTS, self.organization.id, + total_checksum) + assert status == ChunkFileState.ERROR + + def test_artifacts_invalid_release(self): + bundle_file = self.create_artifact_bundle(release='invalid') + blob1 = FileBlob.from_file(ContentFile(bundle_file)) + total_checksum = sha1(bundle_file).hexdigest() + + assemble_artifacts( + org_id=self.organization.id, + version=self.release.version, + checksum=total_checksum, + chunks=[blob1.checksum], + ) + + status, details = get_assemble_status(AssembleTask.ARTIFACTS, self.organization.id, + total_checksum) + assert status == ChunkFileState.ERROR + + def test_artifacts_invalid_zip(self): + bundle_file = b'' + blob1 = FileBlob.from_file(ContentFile(bundle_file)) + total_checksum = sha1(bundle_file).hexdigest() + + assemble_artifacts( + org_id=self.organization.id, + version=self.release.version, + checksum=total_checksum, + chunks=[blob1.checksum], + ) + + status, details = get_assemble_status(AssembleTask.ARTIFACTS, self.organization.id, + total_checksum) + assert status == ChunkFileState.ERROR