From b557bb3d08f362c0d0c0c03371b7a25b029da52f Mon Sep 17 00:00:00 2001 From: Lauryn Brown Date: Wed, 22 May 2019 16:59:48 -0700 Subject: [PATCH 1/8] saving place --- .../api/endpoints/organization_events.py | 24 +++++ src/sentry/api/urls.py | 7 +- .../api/endpoints/test_organization_events.py | 36 +++++++- .../test_organization_tagkey_values.py | 9 ++ .../api/endpoints/test_organization_tags.py | 90 ++++++++++++++----- 5 files changed, 143 insertions(+), 23 deletions(-) diff --git a/src/sentry/api/endpoints/organization_events.py b/src/sentry/api/endpoints/organization_events.py index d6138c4ab90b87..e69d93f7d122c2 100644 --- a/src/sentry/api/endpoints/organization_events.py +++ b/src/sentry/api/endpoints/organization_events.py @@ -137,6 +137,30 @@ def get(self, request, organization): ) +class OrganizationEventsTagsEndpoint(OrganizationEventsEndpointBase): + def get(self, request, organization): + try: + snuba_args = self.get_snuba_query_args(request, organization) + except OrganizationEventsError as exc: + return Response({'detail': exc.message}, status=400) + except NoProjects: + return Response({'count': 0}) + + keys = request.GET.get('keys') + if not keys: + return Response({'detail': 'Event tag keys must be specified'}, status=400) + snuba_args['filter_keys']['tags_key'] = keys + result = raw_query( + aggregations=[['count()', '', 'count']], + groupby=['tags_value'], + referrer='api.organization-events-tags', + limit=10000, + **snuba_args + ) + + return Response(serialize(result, request.user)) + + class OrganizationEventsMetaEndpoint(OrganizationEventsEndpointBase): def get(self, request, organization): diff --git a/src/sentry/api/urls.py b/src/sentry/api/urls.py index 428e0bcb81fb95..7d276c15237f3c 100644 --- a/src/sentry/api/urls.py +++ b/src/sentry/api/urls.py @@ -71,7 +71,7 @@ from .endpoints.organization_discover_query import OrganizationDiscoverQueryEndpoint from .endpoints.organization_discover_saved_queries import OrganizationDiscoverSavedQueriesEndpoint from .endpoints.organization_discover_saved_query_detail import OrganizationDiscoverSavedQueryDetailEndpoint -from .endpoints.organization_events import OrganizationEventsEndpoint, OrganizationEventsMetaEndpoint, OrganizationEventsStatsEndpoint +from .endpoints.organization_events import OrganizationEventsEndpoint, OrganizationEventsMetaEndpoint, OrganizationEventsStatsEndpoint, OrganizationEventsTagsEndpoint from .endpoints.organization_group_index import OrganizationGroupIndexEndpoint from .endpoints.organization_dashboard_details import OrganizationDashboardDetailsEndpoint from .endpoints.organization_dashboard_widget_details import OrganizationDashboardWidgetDetailsEndpoint @@ -595,6 +595,11 @@ OrganizationEventsStatsEndpoint.as_view(), name='sentry-api-0-organization-events-stats' ), + url( + r'^organizations/(?P[^\/]+)/events-tags/$', + OrganizationEventsTagsEndpoint.as_view(), + name='sentry-api-0-organization-events-tags' + ), url( r'^organizations/(?P[^\/]+)/events-meta/$', OrganizationEventsMetaEndpoint.as_view(), diff --git a/tests/snuba/api/endpoints/test_organization_events.py b/tests/snuba/api/endpoints/test_organization_events.py index 9dbd5ef6a9a416..c40ad47940cb50 100644 --- a/tests/snuba/api/endpoints/test_organization_events.py +++ b/tests/snuba/api/endpoints/test_organization_events.py @@ -5,7 +5,7 @@ from datetime import timedelta from django.utils import timezone from django.core.urlresolvers import reverse - +from uuid import uuid4 from sentry.testutils import APITestCase, SnubaTestCase @@ -778,6 +778,40 @@ def test_with_event_count_flag(self): ] +class OrganizationEventsTagsEndpointTest(OrganizationEventsTestBase): + def setUp(self): + super(OrganizationEventsTagsEndpointTest, self).setUp() + self.login_as(user=self.user) + + self.project = self.create_project() + project2 = self.create_project() + self.group = self.create_group(project=self.project) + group2 = self.create_group(project=project2) + self.create_event(event_id='a' * 32, group=self.group, datetime=self.min_ago) + self.create_event(event_id='m' * 32, group=group2, datetime=self.min_ago) + + def test_simple(self): + for i in range(0, 20): + self.create_event( + event_id=uuid4().hex, group=self.group, datetime=self.day_ago, tags={ + 'num': '%d' % i}, + ) + url = reverse( + 'sentry-api-0-organization-events-tags', + kwargs={ + 'organization_slug': self.project.organization.slug, + } + ) + + response = self.client.get( + '%s?%s' % (url, 'keys=num'), + format='json' + ) + assert response.status_code == 200, response.content + # this is not exact because of turbo=True + assert response.data['count'] == 10 + + class OrganizationEventsMetaEndpoint(OrganizationEventsTestBase): def test_simple(self): self.login_as(user=self.user) diff --git a/tests/snuba/api/endpoints/test_organization_tagkey_values.py b/tests/snuba/api/endpoints/test_organization_tagkey_values.py index 75a6f25f0ece36..1be02a7fa853dd 100644 --- a/tests/snuba/api/endpoints/test_organization_tagkey_values.py +++ b/tests/snuba/api/endpoints/test_organization_tagkey_values.py @@ -4,6 +4,7 @@ from django.utils import timezone from django.core.urlresolvers import reverse from exam import fixture +from uuid import uuid4 from sentry.testutils import APITestCase, SnubaTestCase @@ -132,3 +133,11 @@ def test_array_column(self): def test_no_projects(self): self.run_test('fruit', expected=[]) + + def test_large_number_categories(self): + for i in range(0, 20): + self.create_event( + event_id=uuid4().hex, group=self.group, datetime=self.day_ago, tags={ + 'num': '%d' % i}, + ) + self.run_test('num', expected=[('3', 1), ('2', 1), ('1', 2)]) diff --git a/tests/snuba/api/endpoints/test_organization_tags.py b/tests/snuba/api/endpoints/test_organization_tags.py index f037367c5e650d..d57cdb7807da18 100644 --- a/tests/snuba/api/endpoints/test_organization_tags.py +++ b/tests/snuba/api/endpoints/test_organization_tags.py @@ -3,6 +3,7 @@ from datetime import timedelta from django.utils import timezone from django.core.urlresolvers import reverse +from uuid import uuid4 from sentry.testutils import APITestCase, SnubaTestCase @@ -11,16 +12,21 @@ class OrganizationTagsTest(APITestCase, SnubaTestCase): def setUp(self): super(OrganizationTagsTest, self).setUp() self.min_ago = timezone.now() - timedelta(minutes=1) + self.user = self.create_user() + self.login_as(user=self.user) + self.org = self.create_organization() + self.team = self.create_team(organization=self.org) + self.create_member(organization=self.org, user=self.user, teams=[self.team]) - def test_simple(self): - user = self.create_user() - org = self.create_organization() - team = self.create_team(organization=org) - self.create_member(organization=org, user=user, teams=[team]) - - self.login_as(user=user) + self.url = reverse( + 'sentry-api-0-organization-tags', + kwargs={ + 'organization_slug': self.org.slug, + } + ) - project = self.create_project(organization=org, teams=[team]) + def test_simple(self): + project = self.create_project(organization=self.org, teams=[self.team]) group = self.create_group(project=project) self.create_event( @@ -36,14 +42,7 @@ def test_simple(self): event_id='d' * 32, group=group, datetime=self.min_ago, tags={'fruit': 'orange'} ) - url = reverse( - 'sentry-api-0-organization-tags', - kwargs={ - 'organization_slug': org.slug, - } - ) - - response = self.client.get(url, format='json') + response = self.client.get(self.url, format='json') assert response.status_code == 200, response.content data = response.data data.sort(key=lambda val: val['totalValues'], reverse=True) @@ -53,17 +52,66 @@ def test_simple(self): ] def test_no_projects(self): - user = self.create_user() - org = self.create_organization(owner=user) - self.login_as(user=user) - + org = self.create_organization(owner=self.user) url = reverse( 'sentry-api-0-organization-tags', kwargs={ 'organization_slug': org.slug, } ) - response = self.client.get(url, format='json') assert response.status_code == 200, response.content assert response.data == [] + + def create_event_with_tags(self, group, tags): + self.create_event( + event_id=uuid4().hex, group=group, datetime=self.min_ago, tags=tags + ) + + def test_selected_projects(self): + project = self.create_project(organization=self.org, teams=[self.team]) + + group1 = self.create_group(project=project) + group1_tags = [{'fruit': 'apple'}, {'fruit': 'orange'}, + {'some_tag': 'some_value'}, {'fruit': 'orange'}] + for tags in group1_tags: + self.create_event_with_tags( + group=group1, tags=tags + ) + + project2 = self.create_project(organization=self.org, teams=[self.team]) + group2 = self.create_group(project=project2) + group2_tags = [{'fruit': 'apple'}, + {'some_tag': 'some_value'}, + {'fruit': 'orange'}, + {'uniq_tag': 'blah'}] + for tags in group2_tags: + self.create_event_with_tags( + group=group2, tags=tags + ) + + response = self.client.get( + '%s?project=%d&project=%d' % (self.url, project2.id, project.id), + format='json' + ) + assert response.status_code == 200, response.content + data = response.data + data.sort(key=lambda val: val['totalValues'], reverse=True) + assert data == [ + {'name': 'Fruit', 'key': 'fruit', 'totalValues': 5}, + {'name': 'Some Tag', 'key': 'some_tag', 'totalValues': 2}, + {'name': 'Uniq Tag', 'key': 'uniq_tag', 'totalValues': 1} + ] + + response = self.client.get( + '%s?project=%d' % (self.url, project2.id), + format='json' + ) + assert response.status_code == 200, response.content + data = response.data + data.sort(key=lambda val: val['totalValues'], reverse=True) + assert data == [ + {'name': 'Fruit', 'key': 'fruit', 'totalValues': 2}, + {'name': 'Uniq Tag', 'key': 'uniq_tag', 'totalValues': 1}, + {'name': 'Some Tag', 'key': 'some_tag', 'totalValues': 1}, + ] From 96d49f9851f0401191a6998c03f5ffa23c4b013c Mon Sep 17 00:00:00 2001 From: Lauryn Brown Date: Thu, 23 May 2019 17:25:09 -0700 Subject: [PATCH 2/8] got events-tag working for a single key. --- .../api/endpoints/organization_events.py | 29 ++--- src/sentry/api/urls.py | 2 +- src/sentry/tagstore/snuba/backend.py | 23 ++-- .../api/endpoints/test_organization_events.py | 112 ++++++++++++++---- 4 files changed, 124 insertions(+), 42 deletions(-) diff --git a/src/sentry/api/endpoints/organization_events.py b/src/sentry/api/endpoints/organization_events.py index e69d93f7d122c2..a0f921da2a4612 100644 --- a/src/sentry/api/endpoints/organization_events.py +++ b/src/sentry/api/endpoints/organization_events.py @@ -5,7 +5,9 @@ from rest_framework.response import Response +from sentry import tagstore from sentry.api.bases import OrganizationEventsEndpointBase, OrganizationEventsError, NoProjects +from sentry.api.exceptions import ResourceDoesNotExist from sentry.api.helpers.events import get_direct_hit_response from sentry.api.paginator import GenericOffsetPaginator from sentry.api.serializers import EventSerializer, serialize, SimpleEventSerializer @@ -138,27 +140,26 @@ def get(self, request, organization): class OrganizationEventsTagsEndpoint(OrganizationEventsEndpointBase): - def get(self, request, organization): + def get(self, request, organization, key): try: snuba_args = self.get_snuba_query_args(request, organization) except OrganizationEventsError as exc: return Response({'detail': exc.message}, status=400) except NoProjects: - return Response({'count': 0}) + return Response({'detail': 'A valid project must be included.'}, status=400) - keys = request.GET.get('keys') - if not keys: - return Response({'detail': 'Event tag keys must be specified'}, status=400) - snuba_args['filter_keys']['tags_key'] = keys - result = raw_query( - aggregations=[['count()', '', 'count']], - groupby=['tags_value'], - referrer='api.organization-events-tags', - limit=10000, - **snuba_args - ) + lookup_key = tagstore.prefix_reserved_key(key) + project_ids = snuba_args['filter_keys']['project_id'] + environment_ids = snuba_args['filter_keys'].get('environment_id') + aggregations = snuba_args.get('aggregations') + conditions = snuba_args.get('conditions') + try: + tag_key = tagstore.get_tag_key( + project_ids, environment_ids, lookup_key, conditions=conditions, aggregations=aggregations) + except tagstore.TagKeyNotFound: + raise ResourceDoesNotExist - return Response(serialize(result, request.user)) + return Response(serialize(tag_key, request.user)) class OrganizationEventsMetaEndpoint(OrganizationEventsEndpointBase): diff --git a/src/sentry/api/urls.py b/src/sentry/api/urls.py index 7d276c15237f3c..ddb9d95d989b86 100644 --- a/src/sentry/api/urls.py +++ b/src/sentry/api/urls.py @@ -596,7 +596,7 @@ name='sentry-api-0-organization-events-stats' ), url( - r'^organizations/(?P[^\/]+)/events-tags/$', + r'^organizations/(?P[^\/]+)/events-tags/(?P[^/]+)/$', OrganizationEventsTagsEndpoint.as_view(), name='sentry-api-0-organization-events-tags' ), diff --git a/src/sentry/tagstore/snuba/backend.py b/src/sentry/tagstore/snuba/backend.py index 500388dd66a88c..cf151340c0ff5d 100644 --- a/src/sentry/tagstore/snuba/backend.py +++ b/src/sentry/tagstore/snuba/backend.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import functools -from collections import defaultdict +from collections import defaultdict, Iterable from datetime import timedelta from dateutil.parser import parse as parse_datetime from django.utils import timezone @@ -100,18 +100,25 @@ def __get_tag_key(self, project_id, group_id, environment_id, key): return GroupTagKey(group_id=group_id, **data) def __get_tag_key_and_top_values(self, project_id, group_id, environment_id, - key, limit=3, raise_on_empty=True): + key, limit=3, raise_on_empty=True, + conditions=None, aggregations=None): + start, end = self.get_time_range() tag = u'tags[{}]'.format(key) filters = { - 'project_id': [project_id], + 'project_id': project_id if isinstance(project_id, Iterable) else [project_id], } if environment_id: filters['environment'] = [environment_id] if group_id is not None: filters['issue'] = [group_id] - conditions = [[tag, '!=', '']] - aggregations = [ + if not conditions: + conditions = [] + if not aggregations: + aggregations = [] + + conditions.append([tag, '!=', '']) + aggregations += [ ['uniq', tag, 'values_seen'], ['count()', '', 'count'], ['min', SEEN_COLUMN, 'first_seen'], @@ -244,9 +251,11 @@ def __get_tag_value(self, project_id, group_id, environment_id, key, value): else: return GroupTagValue(group_id=group_id, **fix_tag_value_data(data)) - def get_tag_key(self, project_id, environment_id, key, status=TagKeyStatus.VISIBLE): + def get_tag_key(self, project_id, environment_id, key, status=TagKeyStatus.VISIBLE, + conditions=None, aggregations=None): assert status is TagKeyStatus.VISIBLE - return self.__get_tag_key_and_top_values(project_id, None, environment_id, key) + return self.__get_tag_key_and_top_values( + project_id, None, environment_id, key, conditions=conditions, aggregations=aggregations) def get_tag_keys( self, project_id, environment_id, status=TagKeyStatus.VISIBLE, diff --git a/tests/snuba/api/endpoints/test_organization_events.py b/tests/snuba/api/endpoints/test_organization_events.py index c40ad47940cb50..1db3994c3b5a6f 100644 --- a/tests/snuba/api/endpoints/test_organization_events.py +++ b/tests/snuba/api/endpoints/test_organization_events.py @@ -5,7 +5,7 @@ from datetime import timedelta from django.utils import timezone from django.core.urlresolvers import reverse -from uuid import uuid4 + from sentry.testutils import APITestCase, SnubaTestCase @@ -782,34 +782,106 @@ class OrganizationEventsTagsEndpointTest(OrganizationEventsTestBase): def setUp(self): super(OrganizationEventsTagsEndpointTest, self).setUp() self.login_as(user=self.user) - self.project = self.create_project() - project2 = self.create_project() - self.group = self.create_group(project=self.project) - group2 = self.create_group(project=project2) - self.create_event(event_id='a' * 32, group=self.group, datetime=self.min_ago) - self.create_event(event_id='m' * 32, group=group2, datetime=self.min_ago) - - def test_simple(self): - for i in range(0, 20): - self.create_event( - event_id=uuid4().hex, group=self.group, datetime=self.day_ago, tags={ - 'num': '%d' % i}, - ) - url = reverse( + self.url = reverse( 'sentry-api-0-organization-events-tags', kwargs={ 'organization_slug': self.project.organization.slug, + 'key': 'color', } ) - response = self.client.get( - '%s?%s' % (url, 'keys=num'), - format='json' + def assert_top_values(self, output, expected): + assert len(output) == len(expected) + for index, (count, value) in enumerate(expected): + assert output[index]['count'] == count + assert output[index]['value'] == value + + def test_simple(self): + group = self.create_group(project=self.project) + self.create_event( + event_id='w' * 32, + group=group, + message="how to make fast", + datetime=self.min_ago, + tags={'world': 'hello'}, ) + self.create_event( + event_id='x' * 32, + group=group, + message="how to make fast", + datetime=self.min_ago, + tags={'color': 'yellow'}, + ) + self.create_event( + event_id='y' * 32, + group=group, + message="Delet the Data", + datetime=self.min_ago, + tags={'color': 'red'}, + ) + self.create_event( + event_id='z' * 32, + group=group, + message="Data the Delet ", + datetime=self.min_ago, + tags={'color': 'yellow'}, + ) + + response = self.client.get(self.url, format='json') + assert response.status_code == 200, response.content - # this is not exact because of turbo=True - assert response.data['count'] == 10 + assert response.data['uniqueValues'] == 2 + assert response.data['name'] == 'Color' + assert response.data['key'] == 'color' + + self.assert_top_values(response.data['topValues'], [(2, 'yellow'), (1, 'red')]) + + def test_tags_with_query(self): + project = self.create_project() + group = self.create_group(project=project) + self.create_event( + event_id='x' * 32, + group=group, + message="how to make fast", + datetime=self.min_ago, + tags={'color': 'green'}, + ) + self.create_event( + event_id='y' * 32, + group=group, + message="Delet the Data", + datetime=self.min_ago, + tags={'color': 'red'}, + ) + self.create_event( + event_id='z' * 32, + group=group, + message="Data the Delet ", + datetime=self.min_ago, + tags={'color': 'yellow'}, + ) + + response = self.client.get(self.url, {'query': 'delet'}, format='json') + + assert response.status_code == 200, response.content + assert response.data['uniqueValues'] == 2 + assert response.data['name'] == 'Color' + assert response.data['key'] == 'color' + self.assert_top_values(response.data['topValues'], [(1, 'yellow'), (1, 'red')]) + + def test_no_projects(self): + org = self.create_organization(owner=self.user) + url = reverse( + 'sentry-api-0-organization-events-tags', + kwargs={ + 'organization_slug': org.slug, + 'key': 'color', + } + ) + response = self.client.get(url, format='json') + assert response.status_code == 400, response.content + assert response.data == {'detail': 'A valid project must be included.'} class OrganizationEventsMetaEndpoint(OrganizationEventsTestBase): From 93c0b49cec19220316fbe23abca4ab14b1004c48 Mon Sep 17 00:00:00 2001 From: Lauryn Brown Date: Thu, 23 May 2019 17:27:41 -0700 Subject: [PATCH 3/8] removed changes to unrelated tags tests. --- .../test_organization_tagkey_values.py | 9 -- .../api/endpoints/test_organization_tags.py | 90 +++++-------------- 2 files changed, 21 insertions(+), 78 deletions(-) diff --git a/tests/snuba/api/endpoints/test_organization_tagkey_values.py b/tests/snuba/api/endpoints/test_organization_tagkey_values.py index 1be02a7fa853dd..75a6f25f0ece36 100644 --- a/tests/snuba/api/endpoints/test_organization_tagkey_values.py +++ b/tests/snuba/api/endpoints/test_organization_tagkey_values.py @@ -4,7 +4,6 @@ from django.utils import timezone from django.core.urlresolvers import reverse from exam import fixture -from uuid import uuid4 from sentry.testutils import APITestCase, SnubaTestCase @@ -133,11 +132,3 @@ def test_array_column(self): def test_no_projects(self): self.run_test('fruit', expected=[]) - - def test_large_number_categories(self): - for i in range(0, 20): - self.create_event( - event_id=uuid4().hex, group=self.group, datetime=self.day_ago, tags={ - 'num': '%d' % i}, - ) - self.run_test('num', expected=[('3', 1), ('2', 1), ('1', 2)]) diff --git a/tests/snuba/api/endpoints/test_organization_tags.py b/tests/snuba/api/endpoints/test_organization_tags.py index d57cdb7807da18..f037367c5e650d 100644 --- a/tests/snuba/api/endpoints/test_organization_tags.py +++ b/tests/snuba/api/endpoints/test_organization_tags.py @@ -3,7 +3,6 @@ from datetime import timedelta from django.utils import timezone from django.core.urlresolvers import reverse -from uuid import uuid4 from sentry.testutils import APITestCase, SnubaTestCase @@ -12,21 +11,16 @@ class OrganizationTagsTest(APITestCase, SnubaTestCase): def setUp(self): super(OrganizationTagsTest, self).setUp() self.min_ago = timezone.now() - timedelta(minutes=1) - self.user = self.create_user() - self.login_as(user=self.user) - self.org = self.create_organization() - self.team = self.create_team(organization=self.org) - self.create_member(organization=self.org, user=self.user, teams=[self.team]) - - self.url = reverse( - 'sentry-api-0-organization-tags', - kwargs={ - 'organization_slug': self.org.slug, - } - ) def test_simple(self): - project = self.create_project(organization=self.org, teams=[self.team]) + user = self.create_user() + org = self.create_organization() + team = self.create_team(organization=org) + self.create_member(organization=org, user=user, teams=[team]) + + self.login_as(user=user) + + project = self.create_project(organization=org, teams=[team]) group = self.create_group(project=project) self.create_event( @@ -42,7 +36,14 @@ def test_simple(self): event_id='d' * 32, group=group, datetime=self.min_ago, tags={'fruit': 'orange'} ) - response = self.client.get(self.url, format='json') + url = reverse( + 'sentry-api-0-organization-tags', + kwargs={ + 'organization_slug': org.slug, + } + ) + + response = self.client.get(url, format='json') assert response.status_code == 200, response.content data = response.data data.sort(key=lambda val: val['totalValues'], reverse=True) @@ -52,66 +53,17 @@ def test_simple(self): ] def test_no_projects(self): - org = self.create_organization(owner=self.user) + user = self.create_user() + org = self.create_organization(owner=user) + self.login_as(user=user) + url = reverse( 'sentry-api-0-organization-tags', kwargs={ 'organization_slug': org.slug, } ) + response = self.client.get(url, format='json') assert response.status_code == 200, response.content assert response.data == [] - - def create_event_with_tags(self, group, tags): - self.create_event( - event_id=uuid4().hex, group=group, datetime=self.min_ago, tags=tags - ) - - def test_selected_projects(self): - project = self.create_project(organization=self.org, teams=[self.team]) - - group1 = self.create_group(project=project) - group1_tags = [{'fruit': 'apple'}, {'fruit': 'orange'}, - {'some_tag': 'some_value'}, {'fruit': 'orange'}] - for tags in group1_tags: - self.create_event_with_tags( - group=group1, tags=tags - ) - - project2 = self.create_project(organization=self.org, teams=[self.team]) - group2 = self.create_group(project=project2) - group2_tags = [{'fruit': 'apple'}, - {'some_tag': 'some_value'}, - {'fruit': 'orange'}, - {'uniq_tag': 'blah'}] - for tags in group2_tags: - self.create_event_with_tags( - group=group2, tags=tags - ) - - response = self.client.get( - '%s?project=%d&project=%d' % (self.url, project2.id, project.id), - format='json' - ) - assert response.status_code == 200, response.content - data = response.data - data.sort(key=lambda val: val['totalValues'], reverse=True) - assert data == [ - {'name': 'Fruit', 'key': 'fruit', 'totalValues': 5}, - {'name': 'Some Tag', 'key': 'some_tag', 'totalValues': 2}, - {'name': 'Uniq Tag', 'key': 'uniq_tag', 'totalValues': 1} - ] - - response = self.client.get( - '%s?project=%d' % (self.url, project2.id), - format='json' - ) - assert response.status_code == 200, response.content - data = response.data - data.sort(key=lambda val: val['totalValues'], reverse=True) - assert data == [ - {'name': 'Fruit', 'key': 'fruit', 'totalValues': 2}, - {'name': 'Uniq Tag', 'key': 'uniq_tag', 'totalValues': 1}, - {'name': 'Some Tag', 'key': 'some_tag', 'totalValues': 1}, - ] From a835f9ab054b2e1f7b597d9784eaf8118ceac593 Mon Sep 17 00:00:00 2001 From: Lauryn Brown Date: Thu, 23 May 2019 18:32:02 -0700 Subject: [PATCH 4/8] start and end dates --- .../api/endpoints/organization_events.py | 5 +- src/sentry/tagstore/snuba/backend.py | 18 ++--- .../api/endpoints/test_organization_events.py | 65 ++++++++++++++++--- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/sentry/api/endpoints/organization_events.py b/src/sentry/api/endpoints/organization_events.py index a0f921da2a4612..b8d6136e4558d2 100644 --- a/src/sentry/api/endpoints/organization_events.py +++ b/src/sentry/api/endpoints/organization_events.py @@ -151,11 +151,10 @@ def get(self, request, organization, key): lookup_key = tagstore.prefix_reserved_key(key) project_ids = snuba_args['filter_keys']['project_id'] environment_ids = snuba_args['filter_keys'].get('environment_id') - aggregations = snuba_args.get('aggregations') - conditions = snuba_args.get('conditions') + try: tag_key = tagstore.get_tag_key( - project_ids, environment_ids, lookup_key, conditions=conditions, aggregations=aggregations) + project_ids, environment_ids, lookup_key, **snuba_args) except tagstore.TagKeyNotFound: raise ResourceDoesNotExist diff --git a/src/sentry/tagstore/snuba/backend.py b/src/sentry/tagstore/snuba/backend.py index cf151340c0ff5d..98abc7ba40b572 100644 --- a/src/sentry/tagstore/snuba/backend.py +++ b/src/sentry/tagstore/snuba/backend.py @@ -100,10 +100,12 @@ def __get_tag_key(self, project_id, group_id, environment_id, key): return GroupTagKey(group_id=group_id, **data) def __get_tag_key_and_top_values(self, project_id, group_id, environment_id, - key, limit=3, raise_on_empty=True, - conditions=None, aggregations=None): + key, limit=3, raise_on_empty=True, **kwargs): + + default_start, default_end = self.get_time_range() + start = kwargs.get('start', default_start) + end = kwargs.get('end', default_end) - start, end = self.get_time_range() tag = u'tags[{}]'.format(key) filters = { 'project_id': project_id if isinstance(project_id, Iterable) else [project_id], @@ -112,10 +114,8 @@ def __get_tag_key_and_top_values(self, project_id, group_id, environment_id, filters['environment'] = [environment_id] if group_id is not None: filters['issue'] = [group_id] - if not conditions: - conditions = [] - if not aggregations: - aggregations = [] + conditions = kwargs.get('conditions', []) + aggregations = kwargs.get('aggregations', []) conditions.append([tag, '!=', '']) aggregations += [ @@ -252,10 +252,10 @@ def __get_tag_value(self, project_id, group_id, environment_id, key, value): return GroupTagValue(group_id=group_id, **fix_tag_value_data(data)) def get_tag_key(self, project_id, environment_id, key, status=TagKeyStatus.VISIBLE, - conditions=None, aggregations=None): + **kwargs): assert status is TagKeyStatus.VISIBLE return self.__get_tag_key_and_top_values( - project_id, None, environment_id, key, conditions=conditions, aggregations=aggregations) + project_id, None, environment_id, key, **kwargs) def get_tag_keys( self, project_id, environment_id, status=TagKeyStatus.VISIBLE, diff --git a/tests/snuba/api/endpoints/test_organization_events.py b/tests/snuba/api/endpoints/test_organization_events.py index 1db3994c3b5a6f..8b31db4d75bbd7 100644 --- a/tests/snuba/api/endpoints/test_organization_events.py +++ b/tests/snuba/api/endpoints/test_organization_events.py @@ -783,6 +783,7 @@ def setUp(self): super(OrganizationEventsTagsEndpointTest, self).setUp() self.login_as(user=self.user) self.project = self.create_project() + self.group = self.create_group(project=self.project) self.url = reverse( 'sentry-api-0-organization-events-tags', kwargs={ @@ -798,31 +799,30 @@ def assert_top_values(self, output, expected): assert output[index]['value'] == value def test_simple(self): - group = self.create_group(project=self.project) self.create_event( event_id='w' * 32, - group=group, + group=self.group, message="how to make fast", datetime=self.min_ago, tags={'world': 'hello'}, ) self.create_event( event_id='x' * 32, - group=group, + group=self.group, message="how to make fast", datetime=self.min_ago, tags={'color': 'yellow'}, ) self.create_event( event_id='y' * 32, - group=group, + group=self.group, message="Delet the Data", datetime=self.min_ago, tags={'color': 'red'}, ) self.create_event( event_id='z' * 32, - group=group, + group=self.group, message="Data the Delet ", datetime=self.min_ago, tags={'color': 'yellow'}, @@ -838,25 +838,23 @@ def test_simple(self): self.assert_top_values(response.data['topValues'], [(2, 'yellow'), (1, 'red')]) def test_tags_with_query(self): - project = self.create_project() - group = self.create_group(project=project) self.create_event( event_id='x' * 32, - group=group, + group=self.group, message="how to make fast", datetime=self.min_ago, tags={'color': 'green'}, ) self.create_event( event_id='y' * 32, - group=group, + group=self.group, message="Delet the Data", datetime=self.min_ago, tags={'color': 'red'}, ) self.create_event( event_id='z' * 32, - group=group, + group=self.group, message="Data the Delet ", datetime=self.min_ago, tags={'color': 'yellow'}, @@ -870,6 +868,53 @@ def test_tags_with_query(self): assert response.data['key'] == 'color' self.assert_top_values(response.data['topValues'], [(1, 'yellow'), (1, 'red')]) + def test_start_end(self): + two_days_ago = self.day_ago - timedelta(days=1) + hour_ago = self.min_ago - timedelta(hours=1) + self.create_event( + event_id='x' * 32, + group=self.group, + message="Delet the Data", + datetime=two_days_ago, + tags={'color': 'red'}, + ) + self.create_event( + event_id='y' * 32, + group=self.group, + message="Delet the Data", + datetime=hour_ago, + tags={'color': 'red'}, + ) + self.create_event( + event_id='z' * 32, + group=self.group, + message="Data the Delet ", + datetime=hour_ago, + tags={'color': 'red'}, + ) + self.create_event( + event_id='a' * 32, + group=self.group, + message="Delet the Data", + datetime=timezone.now(), + tags={'color': 'red'}, + ) + + response = self.client.get( + self.url, + { + 'start': self.day_ago.isoformat()[:19], + 'end': self.min_ago.isoformat()[:19], + }, + format='json' + ) + + assert response.status_code == 200, response.content + assert response.data['uniqueValues'] == 1 + assert response.data['name'] == 'Color' + assert response.data['key'] == 'color' + self.assert_top_values(response.data['topValues'], [(2, 'red')]) + def test_no_projects(self): org = self.create_organization(owner=self.user) url = reverse( From 4ae9ab31d2c144bc76ebff71f5357e571f150aad Mon Sep 17 00:00:00 2001 From: Lauryn Brown Date: Tue, 28 May 2019 12:33:39 -0700 Subject: [PATCH 5/8] revised to work for multiple keys. --- .../api/endpoints/organization_events.py | 13 +- src/sentry/api/urls.py | 8 +- src/sentry/tagstore/snuba/backend.py | 39 +-- .../api/endpoints/test_organization_events.py | 238 +++++++++++++++--- 4 files changed, 242 insertions(+), 56 deletions(-) diff --git a/src/sentry/api/endpoints/organization_events.py b/src/sentry/api/endpoints/organization_events.py index b8d6136e4558d2..6e462e74b4f210 100644 --- a/src/sentry/api/endpoints/organization_events.py +++ b/src/sentry/api/endpoints/organization_events.py @@ -139,8 +139,8 @@ def get(self, request, organization): ) -class OrganizationEventsTagsEndpoint(OrganizationEventsEndpointBase): - def get(self, request, organization, key): +class OrganizationEventsHeatmapEndpoint(OrganizationEventsEndpointBase): + def get(self, request, organization): try: snuba_args = self.get_snuba_query_args(request, organization) except OrganizationEventsError as exc: @@ -148,13 +148,16 @@ def get(self, request, organization, key): except NoProjects: return Response({'detail': 'A valid project must be included.'}, status=400) - lookup_key = tagstore.prefix_reserved_key(key) + lookup_keys = [tagstore.prefix_reserved_key(key) for key in request.GET.getlist('keys')] + + if not lookup_keys: + return Response({'detail': 'Tag keys must be sepcified.'}, status=400) project_ids = snuba_args['filter_keys']['project_id'] environment_ids = snuba_args['filter_keys'].get('environment_id') try: - tag_key = tagstore.get_tag_key( - project_ids, environment_ids, lookup_key, **snuba_args) + tag_key = tagstore.get_group_tag_keys_and_top_values( + project_ids, None, environment_ids, keys=lookup_keys, get_excluded_tags=True, **snuba_args) except tagstore.TagKeyNotFound: raise ResourceDoesNotExist diff --git a/src/sentry/api/urls.py b/src/sentry/api/urls.py index ddb9d95d989b86..63f4c31bef66f2 100644 --- a/src/sentry/api/urls.py +++ b/src/sentry/api/urls.py @@ -71,7 +71,7 @@ from .endpoints.organization_discover_query import OrganizationDiscoverQueryEndpoint from .endpoints.organization_discover_saved_queries import OrganizationDiscoverSavedQueriesEndpoint from .endpoints.organization_discover_saved_query_detail import OrganizationDiscoverSavedQueryDetailEndpoint -from .endpoints.organization_events import OrganizationEventsEndpoint, OrganizationEventsMetaEndpoint, OrganizationEventsStatsEndpoint, OrganizationEventsTagsEndpoint +from .endpoints.organization_events import OrganizationEventsEndpoint, OrganizationEventsMetaEndpoint, OrganizationEventsStatsEndpoint, OrganizationEventsHeatmapEndpoint from .endpoints.organization_group_index import OrganizationGroupIndexEndpoint from .endpoints.organization_dashboard_details import OrganizationDashboardDetailsEndpoint from .endpoints.organization_dashboard_widget_details import OrganizationDashboardWidgetDetailsEndpoint @@ -596,9 +596,9 @@ name='sentry-api-0-organization-events-stats' ), url( - r'^organizations/(?P[^\/]+)/events-tags/(?P[^/]+)/$', - OrganizationEventsTagsEndpoint.as_view(), - name='sentry-api-0-organization-events-tags' + r'^organizations/(?P[^\/]+)/events-heatmap/$', + OrganizationEventsHeatmapEndpoint.as_view(), + name='sentry-api-0-organization-events-heatmap' ), url( r'^organizations/(?P[^\/]+)/events-meta/$', diff --git a/src/sentry/tagstore/snuba/backend.py b/src/sentry/tagstore/snuba/backend.py index 98abc7ba40b572..79d79bb6627803 100644 --- a/src/sentry/tagstore/snuba/backend.py +++ b/src/sentry/tagstore/snuba/backend.py @@ -47,6 +47,10 @@ def fix_tag_value_data(data): return data +def get_project_list(project_id): + return project_id if isinstance(project_id, Iterable) else [project_id] + + class SnubaTagStorage(TagStorage): # These keys correspond to tags that are typically prefixed with `sentry:` @@ -72,7 +76,7 @@ def __get_tag_key(self, project_id, group_id, environment_id, key): start, end = self.get_time_range() tag = u'tags[{}]'.format(key) filters = { - 'project_id': [project_id], + 'project_id': get_project_list(project_id), } if environment_id: filters['environment'] = [environment_id] @@ -108,7 +112,7 @@ def __get_tag_key_and_top_values(self, project_id, group_id, environment_id, tag = u'tags[{}]'.format(key) filters = { - 'project_id': project_id if isinstance(project_id, Iterable) else [project_id], + 'project_id': get_project_list(project_id), } if environment_id: filters['environment'] = [environment_id] @@ -164,7 +168,7 @@ def __get_tag_keys( ): start, end = self.get_time_range() return self.__get_tag_keys_for_projects( - [project_id], + get_project_list(project_id), group_id, environment_ids, start, @@ -224,7 +228,7 @@ def __get_tag_value(self, project_id, group_id, environment_id, key, value): start, end = self.get_time_range() tag = u'tags[{}]'.format(key) filters = { - 'project_id': [project_id], + 'project_id': get_project_list(project_id), } if environment_id: filters['environment'] = [environment_id] @@ -368,7 +372,7 @@ def get_group_tag_value_count(self, project_id, group_id, environment_id, key): start, end = self.get_time_range() tag = u'tags[{}]'.format(key) filters = { - 'project_id': [project_id], + 'project_id': get_project_list(project_id), 'issue': [group_id], } if environment_id: @@ -385,19 +389,22 @@ def get_top_group_tag_values(self, project_id, group_id, return tag.top_values def get_group_tag_keys_and_top_values( - self, project_id, group_id, environment_ids, user=None, keys=None, value_limit=TOP_VALUES_DEFAULT_LIMIT): + self, project_id, group_id, environment_ids, user=None, keys=None, value_limit=TOP_VALUES_DEFAULT_LIMIT, + **kwargs): # Similar to __get_tag_key_and_top_values except we get the top values # for all the keys provided. value_limit in this case means the number # of top values for each key, so the total rows returned should be # num_keys * limit. - start, end = self.get_time_range() + default_start, default_end = self.get_time_range() + start = kwargs.get('start', default_start) + end = kwargs.get('end', default_end) # First get totals and unique counts by key. keys_with_counts = self.get_group_tag_keys(project_id, group_id, environment_ids, keys=keys) # Then get the top values with first_seen/last_seen/count for each filters = { - 'project_id': [project_id], + 'project_id': get_project_list(project_id), } if environment_ids: filters['environment'] = environment_ids @@ -405,13 +412,15 @@ def get_group_tag_keys_and_top_values( filters['tags_key'] = keys if group_id is not None: filters['issue'] = [group_id] - - aggregations = [ + conditions = kwargs.get('conditions', []) + aggregations = kwargs.get('aggregations', []) + aggregations += [ ['count()', '', 'count'], ['min', SEEN_COLUMN, 'first_seen'], ['max', SEEN_COLUMN, 'last_seen'], ] - conditions = [['tags_key', 'NOT IN', self.EXCLUDE_TAG_KEYS]] + if not kwargs.get('get_excluded_tags'): + conditions.append(['tags_key', 'NOT IN', self.EXCLUDE_TAG_KEYS]) values_by_key = snuba.query( start, end, ['tags_key', 'tags_value'], conditions, filters, aggregations, @@ -443,7 +452,7 @@ def get_group_tag_keys_and_top_values( def __get_release(self, project_id, group_id, first=True): start, end = self.get_time_range() filters = { - 'project_id': [project_id], + 'project_id': get_project_list(project_id), } conditions = [['tags[sentry:release]', 'IS NOT NULL', None]] if group_id is not None: @@ -566,7 +575,7 @@ def get_tag_value_paginator(self, project_id, environment_id, key, query=None, order_by='-last_seen'): start, end = self.get_time_range() return self.get_tag_value_paginator_for_projects( - [project_id], + get_project_list(project_id), [environment_id] if environment_id else None, key, start, @@ -636,7 +645,7 @@ def get_tag_value_paginator_for_projects(self, projects, environments, key, star def get_group_tag_value_iter(self, project_id, group_id, environment_id, key, callbacks=()): start, end = self.get_time_range() filters = { - 'project_id': [project_id], + 'project_id': get_project_list(project_id), 'tags_key': [key], 'issue': [group_id], } @@ -707,7 +716,7 @@ def get_group_event_filter(self, project_id, group_id, environment_ids, tags, st end = min(end, default_end) if end else default_end filters = { - 'project_id': [project_id], + 'project_id': get_project_list(project_id), 'issue': [group_id], } if environment_ids: diff --git a/tests/snuba/api/endpoints/test_organization_events.py b/tests/snuba/api/endpoints/test_organization_events.py index 8b31db4d75bbd7..eae471deb20d23 100644 --- a/tests/snuba/api/endpoints/test_organization_events.py +++ b/tests/snuba/api/endpoints/test_organization_events.py @@ -778,27 +778,95 @@ def test_with_event_count_flag(self): ] -class OrganizationEventsTagsEndpointTest(OrganizationEventsTestBase): +class OrganizationEventsHeatmapEndpointTest(OrganizationEventsTestBase): def setUp(self): - super(OrganizationEventsTagsEndpointTest, self).setUp() + super(OrganizationEventsHeatmapEndpointTest, self).setUp() self.login_as(user=self.user) self.project = self.create_project() self.group = self.create_group(project=self.project) self.url = reverse( - 'sentry-api-0-organization-events-tags', + 'sentry-api-0-organization-events-heatmap', kwargs={ 'organization_slug': self.project.organization.slug, - 'key': 'color', } ) - - def assert_top_values(self, output, expected): - assert len(output) == len(expected) - for index, (count, value) in enumerate(expected): - assert output[index]['count'] == count - assert output[index]['value'] == value + self.min_ago = self.min_ago.replace(microsecond=0) + self.day_ago = self.day_ago.replace(microsecond=0) def test_simple(self): + self.create_event( + event_id='x' * 32, + group=self.group, + message="how to make fast", + datetime=self.min_ago, + tags={'color': 'green'}, + ) + self.create_event( + event_id='y' * 32, + group=self.group, + message="Delet the Data", + datetime=self.min_ago, + tags={'number': 'one'}, + ) + self.create_event( + event_id='z' * 32, + group=self.group, + message="Data the Delet ", + datetime=self.min_ago, + tags={'color': 'green'}, + ) + self.create_event( + event_id='a' * 32, + group=self.group, + message="Data the Delet ", + datetime=self.min_ago, + tags={'color': 'red'}, + ) + + response = self.client.get(self.url, {'keys': ['color', 'number']}, format='json') + + assert response.status_code == 200, response.content + assert len(response.data) == 2 + response.data[0] == { + 'topValues': [ + { + 'count': 1, + 'name': 'one', + 'value': 'one', + 'lastSeen': self.min_ago, + 'key': 'number', + 'firstSeen': self.min_ago + } + ], + 'totalValues': 1, + 'name': 'Number', + 'key': 'number' + } + response.data[1] == { + 'topValues': [ + { + 'count': 2, + 'name': 'green', + 'value': 'green', + 'lastSeen': self.min_ago, + 'key': 'color', + 'firstSeen': self.min_ago + }, + { + 'count': 1, + 'name': 'red', + 'value': 'red', + 'lastSeen': self.min_ago, + 'key': 'color', + 'firstSeen': self.min_ago + } + ], + 'totalValues': 3, + 'name': 'Color', + 'key': 'color' + } + + def test_single_key(self): self.create_event( event_id='w' * 32, group=self.group, @@ -828,16 +896,34 @@ def test_simple(self): tags={'color': 'yellow'}, ) - response = self.client.get(self.url, format='json') - + response = self.client.get(self.url, {'keys': ['color']}, format='json') assert response.status_code == 200, response.content - assert response.data['uniqueValues'] == 2 - assert response.data['name'] == 'Color' - assert response.data['key'] == 'color' - - self.assert_top_values(response.data['topValues'], [(2, 'yellow'), (1, 'red')]) - - def test_tags_with_query(self): + assert len(response.data) == 1 + assert response.data[0] == { + 'topValues': [ + { + 'count': 2, + 'name': 'yellow', + 'value': 'yellow', + 'lastSeen': self.min_ago, + 'key': 'color', + 'firstSeen': self.min_ago + }, + { + 'count': 1, + 'name': 'red', + 'value': 'red', + 'lastSeen': self.min_ago, + 'key': 'color', + 'firstSeen': self.min_ago + } + ], + 'totalValues': 3, + 'name': 'Color', + 'key': 'color' + } + + def test_with_query(self): self.create_event( event_id='x' * 32, group=self.group, @@ -860,17 +946,38 @@ def test_tags_with_query(self): tags={'color': 'yellow'}, ) - response = self.client.get(self.url, {'query': 'delet'}, format='json') + response = self.client.get(self.url, {'query': 'delet', 'keys': ['color']}, format='json') assert response.status_code == 200, response.content - assert response.data['uniqueValues'] == 2 - assert response.data['name'] == 'Color' - assert response.data['key'] == 'color' - self.assert_top_values(response.data['topValues'], [(1, 'yellow'), (1, 'red')]) + assert len(response.data) == 1 + assert response.data[0] == { + 'topValues': [ + { + 'count': 1, + 'name': 'yellow', + 'value': 'yellow', + 'lastSeen': self.min_ago, + 'key': 'color', + 'firstSeen': self.min_ago + }, + { + 'count': 1, + 'name': 'red', + 'value': 'red', + 'lastSeen': self.min_ago, + 'key': 'color', + 'firstSeen': self.min_ago + } + ], + 'totalValues': 3, + 'name': 'Color', + 'key': 'color' + } def test_start_end(self): two_days_ago = self.day_ago - timedelta(days=1) hour_ago = self.min_ago - timedelta(hours=1) + two_hours_ago = hour_ago - timedelta(hours=1) self.create_event( event_id='x' * 32, group=self.group, @@ -889,7 +996,7 @@ def test_start_end(self): event_id='z' * 32, group=self.group, message="Data the Delet ", - datetime=hour_ago, + datetime=two_hours_ago, tags={'color': 'red'}, ) self.create_event( @@ -905,26 +1012,93 @@ def test_start_end(self): { 'start': self.day_ago.isoformat()[:19], 'end': self.min_ago.isoformat()[:19], + 'keys': ['color'], }, format='json' ) assert response.status_code == 200, response.content - assert response.data['uniqueValues'] == 1 - assert response.data['name'] == 'Color' - assert response.data['key'] == 'color' - self.assert_top_values(response.data['topValues'], [(2, 'red')]) + assert len(response.data) == 1 + assert response.data[0] == { + 'topValues': [ + { + 'count': 2, + 'name': 'red', + 'value': 'red', + 'lastSeen': hour_ago, + 'key': 'color', + 'firstSeen': two_hours_ago + } + ], + 'totalValues': 3, + 'name': 'Color', + 'key': 'color' + } + + def test_excluded_tag(self): + self.user = self.create_user() + self.user2 = self.create_user() + self.create_event( + event_id='a' * 32, + group=self.group, + datetime=self.day_ago, + tags={'sentry:user': self.user.email}, + ) + self.create_event( + event_id='b' * 32, + group=self.group, + datetime=self.day_ago, + tags={'sentry:user': self.user2.email}, + ) + self.create_event( + event_id='c' * 32, + group=self.group, + datetime=self.day_ago, + tags={'sentry:user': self.user2.email}, + ) + response = self.client.get( + self.url, + { + 'keys': ['user'], + }, + format='json' + ) + + assert response.status_code == 200, response.content + assert len(response.data) == 1 + assert response.data[0] == { + 'topValues': [ + { + 'count': 2, + 'name': self.user2.email, + 'value': self.user2.email, + 'lastSeen': self.day_ago, + 'key': 'user', + 'firstSeen': self.day_ago + }, + { + 'count': 1, + 'name': self.user.email, + 'value': self.user.email, + 'lastSeen': self.day_ago, + 'key': 'user', + 'firstSeen': self.day_ago + } + ], + 'totalValues': 3, + 'name': 'User', + 'key': 'user' + } def test_no_projects(self): org = self.create_organization(owner=self.user) url = reverse( - 'sentry-api-0-organization-events-tags', + 'sentry-api-0-organization-events-heatmap', kwargs={ 'organization_slug': org.slug, - 'key': 'color', } ) - response = self.client.get(url, format='json') + response = self.client.get(url, {'keys': ['color']}, format='json') assert response.status_code == 400, response.content assert response.data == {'detail': 'A valid project must be included.'} From f3d62363d0a2b9454ecd405acc192053b49b43ac Mon Sep 17 00:00:00 2001 From: Lauryn Brown Date: Tue, 28 May 2019 14:27:09 -0700 Subject: [PATCH 6/8] Added changes that mark suggested and found a bug. --- .../api/endpoints/organization_events.py | 10 + src/sentry/tagstore/snuba/backend.py | 15 +- .../api/endpoints/test_organization_events.py | 294 +++++++++++------- 3 files changed, 193 insertions(+), 126 deletions(-) diff --git a/src/sentry/api/endpoints/organization_events.py b/src/sentry/api/endpoints/organization_events.py index 6e462e74b4f210..c9f160ae02b350 100644 --- a/src/sentry/api/endpoints/organization_events.py +++ b/src/sentry/api/endpoints/organization_events.py @@ -155,6 +155,16 @@ def get(self, request, organization): project_ids = snuba_args['filter_keys']['project_id'] environment_ids = snuba_args['filter_keys'].get('environment_id') + has_global_views = features.has( + 'organizations:global-views', + organization, + actor=request.user) + + if not has_global_views and len(project_ids) > 1: + return Response({ + 'detail': 'You cannot view events from multiple projects.' + }, status=400) + try: tag_key = tagstore.get_group_tag_keys_and_top_values( project_ids, None, environment_ids, keys=lookup_keys, get_excluded_tags=True, **snuba_args) diff --git a/src/sentry/tagstore/snuba/backend.py b/src/sentry/tagstore/snuba/backend.py index 79d79bb6627803..72df0607e6fead 100644 --- a/src/sentry/tagstore/snuba/backend.py +++ b/src/sentry/tagstore/snuba/backend.py @@ -164,9 +164,12 @@ def __get_tag_key_and_top_values(self, project_id, group_id, environment_id, def __get_tag_keys( self, project_id, group_id, environment_ids, limit=1000, keys=None, - include_values_seen=True, + include_values_seen=True, **kwargs ): - start, end = self.get_time_range() + default_start, default_end = self.get_time_range() + start = kwargs.get('start', default_start) + end = kwargs.get('end', default_end) + return self.__get_tag_keys_for_projects( get_project_list(project_id), group_id, @@ -296,10 +299,11 @@ def get_group_tag_key(self, project_id, group_id, environment_id, key): return self.__get_tag_key_and_top_values( project_id, group_id, environment_id, key, limit=TOP_VALUES_DEFAULT_LIMIT) - def get_group_tag_keys(self, project_id, group_id, environment_ids, limit=None, keys=None): + def get_group_tag_keys(self, project_id, group_id, environment_ids, + limit=None, keys=None, **kwargs): return self.__get_tag_keys( project_id, group_id, environment_ids, limit=limit, keys=keys, - include_values_seen=False, + include_values_seen=False, **kwargs ) def get_group_tag_value(self, project_id, group_id, environment_id, key, value): @@ -400,7 +404,8 @@ def get_group_tag_keys_and_top_values( end = kwargs.get('end', default_end) # First get totals and unique counts by key. - keys_with_counts = self.get_group_tag_keys(project_id, group_id, environment_ids, keys=keys) + keys_with_counts = self.get_group_tag_keys( + project_id, group_id, environment_ids, keys=keys, start=start, end=end) # Then get the top values with first_seen/last_seen/count for each filters = { diff --git a/tests/snuba/api/endpoints/test_organization_events.py b/tests/snuba/api/endpoints/test_organization_events.py index eae471deb20d23..4e5b65173b75de 100644 --- a/tests/snuba/api/endpoints/test_organization_events.py +++ b/tests/snuba/api/endpoints/test_organization_events.py @@ -5,6 +5,7 @@ from datetime import timedelta from django.utils import timezone from django.core.urlresolvers import reverse +from uuid import uuid4 from sentry.testutils import APITestCase, SnubaTestCase @@ -783,7 +784,7 @@ def setUp(self): super(OrganizationEventsHeatmapEndpointTest, self).setUp() self.login_as(user=self.user) self.project = self.create_project() - self.group = self.create_group(project=self.project) + self.project2 = self.create_project() self.url = reverse( 'sentry-api-0-organization-events-heatmap', kwargs={ @@ -794,36 +795,41 @@ def setUp(self): self.day_ago = self.day_ago.replace(microsecond=0) def test_simple(self): - self.create_event( - event_id='x' * 32, - group=self.group, - message="how to make fast", - datetime=self.min_ago, - tags={'color': 'green'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'tags': {'color': 'green'}, + }, + project_id=self.project.id ) - self.create_event( - event_id='y' * 32, - group=self.group, - message="Delet the Data", - datetime=self.min_ago, - tags={'number': 'one'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'tags': {'number': 'one'}, + }, + project_id=self.project2.id ) - self.create_event( - event_id='z' * 32, - group=self.group, - message="Data the Delet ", - datetime=self.min_ago, - tags={'color': 'green'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'tags': {'color': 'green'}, + }, + project_id=self.project.id ) - self.create_event( - event_id='a' * 32, - group=self.group, - message="Data the Delet ", - datetime=self.min_ago, - tags={'color': 'red'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'tags': {'color': 'red'}, + }, + project_id=self.project.id ) - response = self.client.get(self.url, {'keys': ['color', 'number']}, format='json') + with self.feature('organizations:global-views'): + response = self.client.get(self.url, {'keys': ['color', 'number']}, format='json') assert response.status_code == 200, response.content assert len(response.data) == 2 @@ -867,36 +873,41 @@ def test_simple(self): } def test_single_key(self): - self.create_event( - event_id='w' * 32, - group=self.group, - message="how to make fast", - datetime=self.min_ago, - tags={'world': 'hello'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'tags': {'world': 'hello'}, + }, + project_id=self.project.id ) - self.create_event( - event_id='x' * 32, - group=self.group, - message="how to make fast", - datetime=self.min_ago, - tags={'color': 'yellow'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'tags': {'color': 'yellow'}, + }, + project_id=self.project.id ) - self.create_event( - event_id='y' * 32, - group=self.group, - message="Delet the Data", - datetime=self.min_ago, - tags={'color': 'red'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'tags': {'color': 'red'}, + }, + project_id=self.project2.id ) - self.create_event( - event_id='z' * 32, - group=self.group, - message="Data the Delet ", - datetime=self.min_ago, - tags={'color': 'yellow'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'tags': {'color': 'yellow'}, + }, + project_id=self.project.id ) - response = self.client.get(self.url, {'keys': ['color']}, format='json') + with self.feature('organizations:global-views'): + response = self.client.get(self.url, {'keys': ['color']}, format='json') assert response.status_code == 200, response.content assert len(response.data) == 1 assert response.data[0] == { @@ -924,29 +935,38 @@ def test_single_key(self): } def test_with_query(self): - self.create_event( - event_id='x' * 32, - group=self.group, - message="how to make fast", - datetime=self.min_ago, - tags={'color': 'green'}, - ) - self.create_event( - event_id='y' * 32, - group=self.group, - message="Delet the Data", - datetime=self.min_ago, - tags={'color': 'red'}, - ) - self.create_event( - event_id='z' * 32, - group=self.group, - message="Data the Delet ", - datetime=self.min_ago, - tags={'color': 'yellow'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'message': 'how to make fast', + 'tags': {'color': 'green'}, + }, + project_id=self.project.id + ) + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'message': 'Delet the Data', + 'tags': {'color': 'red'}, + }, + project_id=self.project.id + ) + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.min_ago.isoformat(), + 'message': 'Data the Delet ', + 'tags': {'color': 'yellow'}, + }, + project_id=self.project2.id ) - response = self.client.get(self.url, {'query': 'delet', 'keys': ['color']}, format='json') + with self.feature('organizations:global-views'): + response = self.client.get( + self.url, { + 'query': 'delet', 'keys': ['color']}, format='json') assert response.status_code == 200, response.content assert len(response.data) == 1 @@ -978,45 +998,51 @@ def test_start_end(self): two_days_ago = self.day_ago - timedelta(days=1) hour_ago = self.min_ago - timedelta(hours=1) two_hours_ago = hour_ago - timedelta(hours=1) - self.create_event( - event_id='x' * 32, - group=self.group, - message="Delet the Data", - datetime=two_days_ago, - tags={'color': 'red'}, - ) - self.create_event( - event_id='y' * 32, - group=self.group, - message="Delet the Data", - datetime=hour_ago, - tags={'color': 'red'}, + + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': two_days_ago.isoformat(), + 'tags': {'color': 'red'}, + }, + project_id=self.project.id ) - self.create_event( - event_id='z' * 32, - group=self.group, - message="Data the Delet ", - datetime=two_hours_ago, - tags={'color': 'red'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': hour_ago.isoformat(), + 'tags': {'color': 'red'}, + }, + project_id=self.project.id ) - self.create_event( - event_id='a' * 32, - group=self.group, - message="Delet the Data", - datetime=timezone.now(), - tags={'color': 'red'}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': two_hours_ago.isoformat(), + 'tags': {'color': 'red'}, + }, + project_id=self.project.id ) - - response = self.client.get( - self.url, - { - 'start': self.day_ago.isoformat()[:19], - 'end': self.min_ago.isoformat()[:19], - 'keys': ['color'], + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': timezone.now().isoformat(), + 'tags': {'color': 'red'}, }, - format='json' + project_id=self.project2.id ) + with self.feature('organizations:global-views'): + response = self.client.get( + self.url, + { + 'start': self.day_ago.isoformat()[:19], + 'end': self.min_ago.isoformat()[:19], + 'keys': ['color'], + }, + format='json' + ) + assert response.status_code == 200, response.content assert len(response.data) == 1 assert response.data[0] == { @@ -1030,7 +1056,7 @@ def test_start_end(self): 'firstSeen': two_hours_ago } ], - 'totalValues': 3, + 'totalValues': 2, 'name': 'Color', 'key': 'color' } @@ -1038,28 +1064,36 @@ def test_start_end(self): def test_excluded_tag(self): self.user = self.create_user() self.user2 = self.create_user() - self.create_event( - event_id='a' * 32, - group=self.group, - datetime=self.day_ago, - tags={'sentry:user': self.user.email}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.day_ago.isoformat(), + 'tags': {'sentry:user': self.user.email}, + }, + project_id=self.project.id ) - self.create_event( - event_id='b' * 32, - group=self.group, - datetime=self.day_ago, - tags={'sentry:user': self.user2.email}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.day_ago.isoformat(), + 'tags': {'sentry:user': self.user2.email}, + }, + project_id=self.project.id ) - self.create_event( - event_id='c' * 32, - group=self.group, - datetime=self.day_ago, - tags={'sentry:user': self.user2.email}, + self.store_event( + data={ + 'event_id': uuid4().hex, + 'timestamp': self.day_ago.isoformat(), + 'tags': {'sentry:user': self.user2.email}, + }, + project_id=self.project.id ) + response = self.client.get( self.url, { 'keys': ['user'], + 'project': [self.project.id] }, format='json' ) @@ -1102,6 +1136,24 @@ def test_no_projects(self): assert response.status_code == 400, response.content assert response.data == {'detail': 'A valid project must be included.'} + def test_multiple_projects_without_global_view(self): + self.store_event( + data={ + 'event_id': uuid4().hex, + }, + project_id=self.project.id + ) + self.store_event( + data={ + 'event_id': uuid4().hex, + }, + project_id=self.project2.id + ) + + response = self.client.get(self.url, {'keys': ['color']}, format='json') + assert response.status_code == 400, response.content + assert response.data == {'detail': 'You cannot view events from multiple projects.'} + class OrganizationEventsMetaEndpoint(OrganizationEventsTestBase): def test_simple(self): From 027e885a61e6a6dc5119af34f4b7a45275b28e2a Mon Sep 17 00:00:00 2001 From: Lauryn Brown Date: Wed, 29 May 2019 11:19:27 -0700 Subject: [PATCH 7/8] Added **kwargs to base tagstore method. --- src/sentry/tagstore/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/tagstore/base.py b/src/sentry/tagstore/base.py index 2130fe4d4ba989..f500ee3a89022d 100644 --- a/src/sentry/tagstore/base.py +++ b/src/sentry/tagstore/base.py @@ -412,7 +412,7 @@ def update_group_tag_key_values_seen(self, project_id, group_ids): raise NotImplementedError def get_group_tag_keys_and_top_values( - self, project_id, group_id, environment_ids, keys=None, value_limit=TOP_VALUES_DEFAULT_LIMIT): + self, project_id, group_id, environment_ids, keys=None, value_limit=TOP_VALUES_DEFAULT_LIMIT, **kwargs): # only the snuba backend supports multi env, and that overrides this method if environment_ids and len(environment_ids) > 1: From 0dc26809fccb0b6c0b5317028321d6333a80bc79 Mon Sep 17 00:00:00 2001 From: Lauryn Brown Date: Wed, 29 May 2019 14:09:22 -0700 Subject: [PATCH 8/8] fixed typo." --- src/sentry/api/endpoints/organization_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/api/endpoints/organization_events.py b/src/sentry/api/endpoints/organization_events.py index c9f160ae02b350..b004174c8aa783 100644 --- a/src/sentry/api/endpoints/organization_events.py +++ b/src/sentry/api/endpoints/organization_events.py @@ -151,7 +151,7 @@ def get(self, request, organization): lookup_keys = [tagstore.prefix_reserved_key(key) for key in request.GET.getlist('keys')] if not lookup_keys: - return Response({'detail': 'Tag keys must be sepcified.'}, status=400) + return Response({'detail': 'Tag keys must be specified.'}, status=400) project_ids = snuba_args['filter_keys']['project_id'] environment_ids = snuba_args['filter_keys'].get('environment_id')