Merge branch 'master' into armenzg/dev/downgrade-psycopg2-binary

Author: armenzg

.github/CODEOWNERS

@@ -249,8 +249,11 @@ build-utils/        @getsentry/owners-js-build
 /src/sentry/api/endpoints/chunk.py                                  @getsentry/owners-native
 /src/sentry/api/endpoints/project_app_store_connect_credentials.py  @getsentry/owners-native
 /src/sentry/lang/native/                                            @getsentry/owners-native
+src/sentry/processing/realtime_metrics/                             @getsentry/owners-native
 /src/sentry/tasks/app_store_connect.py                              @getsentry/owners-native
 /src/sentry/tasks/assemble.py                                       @getsentry/owners-native
+/src/sentry/tasks/low_priority_symbolication.py                     @getsentry/owners-native
+/tests/sentry/tasks/test_low_priority_symbolication.py              @getsentry/owners-native
 /src/sentry/utils/appleconnect/                                     @getsentry/owners-native
 
 ## End of Native ##

.github/actions/setup-sentry/action.yml

@@ -158,16 +158,12 @@ runs:
 
         # TODO: Use devservices kafka. See https://github.com/getsentry/sentry/pull/20986#issuecomment-704510570
         if [ "$NEED_KAFKA" = "true" ]; then
-          # This is *not* the production version. Unclear reason as to why this was chosen
-          # https://github.com/getsentry/ops/blob/c823e62f930ecc6c97bb08898c71e49edc7232f6/cookbooks/getsentry/attributes/default.rb#L631
           docker run \
             --name sentry_zookeeper \
             -d --network host \
             -e ZOOKEEPER_CLIENT_PORT=2181 \
             confluentinc/cp-zookeeper:4.1.0
 
-          # This is the production version; do not change w/o changing it there as well
-          # https://github.com/getsentry/ops/blob/c823e62f930ecc6c97bb08898c71e49edc7232f6/cookbooks/getsentry/attributes/default.rb#L643
           docker run \
             --name sentry_kafka \
             -d --network host \

docs-ui/stories/components/alertBar.stories.js

@@ -0,0 +1,10 @@
+import PageAlertBar from 'app/components/pageAlertBar';
+
+export default {
+  title: 'Components/Alerts/Alert Bar',
+  component: PageAlertBar,
+};
+
+export const Default = ({...args}) => (
+  <PageAlertBar {...args}>Alert message</PageAlertBar>
+);

mypy.ini

@@ -1,13 +1,16 @@
 [mypy]
 python_version = 3.6
 files = src/sentry/api/bases/external_actor.py,
+        src/sentry/api/bases/organization_events.py,
         src/sentry/api/endpoints/external_team.py,
         src/sentry/api/endpoints/external_team_details.py,
         src/sentry/api/endpoints/external_user.py,
         src/sentry/api/endpoints/external_user_details.py,
         src/sentry/api/endpoints/organization_events_trace.py,
         src/sentry/api/endpoints/project_app_store_connect_credentials.py,
         src/sentry/api/endpoints/project_codeowners.py,
+        src/sentry/api/endpoints/organization_events_stats.py,
+        src/sentry/api/endpoints/team_issue_breakdown.py,
         src/sentry/api/serializers/base.py,
         src/sentry/api/serializers/models/external_actor.py,
         src/sentry/api/serializers/models/integration.py,
@@ -52,6 +55,7 @@ files = src/sentry/api/bases/external_actor.py,
         src/sentry/snuba/query_subscription_consumer.py,
         src/sentry/spans/**/*.py,
         src/sentry/tasks/app_store_connect.py,
+        src/sentry/tasks/low_priority_symbolication.py,
         src/sentry/tasks/update_user_reports.py,
         src/sentry/unmerge.py,
         src/sentry/utils/appleconnect/,

src/sentry/api/bases/organization_events.py

@@ -1,60 +1,56 @@
 from contextlib import contextmanager
-from typing import Sequence
+from datetime import datetime, timedelta
+from typing import Any, Callable, Dict, Generator, Optional, Sequence, Union, cast
 
 import sentry_sdk
-from django.http import HttpRequest
+from django.utils import timezone
 from django.utils.http import urlquote
-from rest_framework.exceptions import APIException, ParseError
+from rest_framework.exceptions import APIException, ParseError, ValidationError
+from rest_framework.request import Request
 from sentry_relay.consts import SPAN_STATUS_CODE_TO_NAME
 
-from sentry import features
+from sentry import features, quotas
 from sentry.api.base import LINK_HEADER
 from sentry.api.bases import NoProjects, OrganizationEndpoint
 from sentry.api.helpers.teams import get_teams
-from sentry.api.serializers.snuba import SnubaTSResultSerializer
+from sentry.api.serializers.snuba import BaseSnubaSerializer, SnubaTSResultSerializer
 from sentry.discover.arithmetic import ArithmeticError, is_equation, strip_equation
 from sentry.exceptions import InvalidSearchQuery
-from sentry.models import Organization, Team
+from sentry.models import Organization, Project, Team
 from sentry.models.group import Group
 from sentry.search.events.constants import TIMEOUT_ERROR_MESSAGE
 from sentry.search.events.fields import get_function_alias
 from sentry.search.events.filter import get_filter
 from sentry.snuba import discover
 from sentry.utils import snuba
+from sentry.utils.cursors import Cursor
 from sentry.utils.dates import get_interval_from_range, get_rollup_from_request, parse_stats_period
 from sentry.utils.http import absolute_uri
-from sentry.utils.snuba import MAX_FIELDS
+from sentry.utils.snuba import MAX_FIELDS, SnubaTSResult
 
 
-def resolve_axis_column(column: str, index=0) -> str:
-    return get_function_alias(column) if not is_equation(column) else f"equation[{index}]"
+def resolve_axis_column(column: str, index: int = 0) -> str:
+    return cast(
+        str, get_function_alias(column) if not is_equation(column) else f"equation[{index}]"
+    )
 
 
-class OrganizationEventsEndpointBase(OrganizationEndpoint):
-    def has_feature(self, organization, request):
+class OrganizationEventsEndpointBase(OrganizationEndpoint):  # type: ignore
+    def has_feature(self, organization: Organization, request: Request) -> bool:
         return features.has(
             "organizations:discover-basic", organization, actor=request.user
         ) or features.has("organizations:performance-view", organization, actor=request.user)
 
-    def get_equation_list(self, organization: Organization, request: HttpRequest) -> Sequence[str]:
+    def get_equation_list(self, organization: Organization, request: Request) -> Sequence[str]:
         """equations have a prefix so that they can be easily included alongside our existing fields"""
         return [
             strip_equation(field) for field in request.GET.getlist("field")[:] if is_equation(field)
         ]
 
-    def get_field_list(self, organization: Organization, request: HttpRequest) -> Sequence[str]:
+    def get_field_list(self, organization: Organization, request: Request) -> Sequence[str]:
         return [field for field in request.GET.getlist("field")[:] if not is_equation(field)]
 
-    def get_snuba_filter(self, request, organization, params=None):
-        if params is None:
-            params = self.get_snuba_params(request, organization)
-        query = request.GET.get("query")
-        try:
-            return get_filter(query, params)
-        except InvalidSearchQuery as e:
-            raise ParseError(detail=str(e))
-
-    def get_team_ids(self, request, organization):
+    def get_team_ids(self, request: Request, organization: Organization) -> Sequence[int]:
         if not request.user:
             return []
 
@@ -64,7 +60,9 @@ def get_team_ids(self, request, organization):
 
         return [team.id for team in teams]
 
-    def get_snuba_params(self, request, organization, check_global_views=True):
+    def get_snuba_params(
+        self, request: Request, organization: Organization, check_global_views: bool = True
+    ) -> Dict[str, Any]:
         with sentry_sdk.start_span(op="discover.endpoint", description="filter_params"):
             if (
                 len(self.get_field_list(organization, request))
@@ -75,7 +73,7 @@ def get_snuba_params(self, request, organization, check_global_views=True):
                     detail=f"You can view up to {MAX_FIELDS} fields at a time. Please delete some and try again."
                 )
 
-            params = self.get_filter_params(request, organization)
+            params: Dict[str, Any] = self.get_filter_params(request, organization)
             params = self.quantize_date_params(request, params)
             params["user_id"] = request.user.id if request.user else None
             params["team_id"] = self.get_team_ids(request, organization)
@@ -89,17 +87,27 @@ def get_snuba_params(self, request, organization, check_global_views=True):
 
             return params
 
-    def get_orderby(self, request):
-        sort = request.GET.getlist("sort")
+    def get_orderby(self, request: Request) -> Optional[Sequence[str]]:
+        sort: Sequence[str] = request.GET.getlist("sort")
         if sort:
             return sort
         # Deprecated. `sort` should be used as it is supported by
         # more endpoints.
-        orderby = request.GET.getlist("orderby")
+        orderby: Sequence[str] = request.GET.getlist("orderby")
         if orderby:
             return orderby
-
-    def get_snuba_query_args_legacy(self, request, organization):
+        return None
+
+    def get_snuba_query_args_legacy(
+        self, request: Request, organization: Organization
+    ) -> Dict[
+        str,
+        Union[
+            Optional[datetime],
+            Sequence[Sequence[Union[str, str, Any]]],
+            Optional[Dict[str, Sequence[int]]],
+        ],
+    ]:
         params = self.get_filter_params(request, organization)
         query = request.GET.get("query")
         try:
@@ -116,7 +124,7 @@ def get_snuba_query_args_legacy(self, request, organization):
 
         return snuba_args
 
-    def quantize_date_params(self, request, params):
+    def quantize_date_params(self, request: Request, params: Dict[str, Any]) -> Dict[str, Any]:
         # We only need to perform this rounding on relative date periods
         if "statsPeriod" not in request.GET:
             return params
@@ -133,7 +141,7 @@ def quantize_date_params(self, request, params):
         return results
 
     @contextmanager
-    def handle_query_errors(self):
+    def handle_query_errors(self) -> Generator[None, None, None]:
         try:
             yield
         except discover.InvalidSearchQuery as error:
@@ -184,7 +192,7 @@ def handle_query_errors(self):
 
 
 class OrganizationEventsV2EndpointBase(OrganizationEventsEndpointBase):
-    def build_cursor_link(self, request, name, cursor):
+    def build_cursor_link(self, request: Request, name: str, cursor: Optional[Cursor]) -> str:
         # The base API function only uses the last query parameter, but this endpoint
         # needs all the parameters, particularly for the "field" query param.
         querystring = "&".join(
@@ -200,21 +208,33 @@ def build_cursor_link(self, request, name, cursor):
         else:
             base_url = base_url + "?"
 
-        return LINK_HEADER.format(
+        return cast(str, LINK_HEADER).format(
             uri=base_url,
             cursor=str(cursor),
             name=name,
             has_results="true" if bool(cursor) else "false",
         )
 
-    def handle_results_with_meta(self, request, organization, project_ids, results):
+    def handle_results_with_meta(
+        self,
+        request: Request,
+        organization: Organization,
+        project_ids: Sequence[int],
+        results: Dict[str, Any],
+    ) -> Dict[str, Any]:
         with sentry_sdk.start_span(op="discover.endpoint", description="base.handle_results"):
             data = self.handle_data(request, organization, project_ids, results.get("data"))
             if not data:
                 return {"data": [], "meta": {}}
             return {"data": data, "meta": results.get("meta", {})}
 
-    def handle_data(self, request, organization, project_ids, results):
+    def handle_data(
+        self,
+        request: Request,
+        organization: Organization,
+        project_ids: Sequence[int],
+        results: Optional[Sequence[Any]],
+    ) -> Optional[Sequence[Any]]:
         if not results:
             return results
 
@@ -240,7 +260,9 @@ def handle_data(self, request, organization, project_ids, results):
 
         return results
 
-    def handle_issues(self, results, project_ids, organization):
+    def handle_issues(
+        self, results: Sequence[Any], project_ids: Sequence[int], organization: Organization
+    ) -> None:
         issue_ids = {row.get("issue.id") for row in results}
         issues = Group.issues_mapping(issue_ids, project_ids, organization)
         for result in results:
@@ -249,16 +271,19 @@ def handle_issues(self, results, project_ids, organization):
 
     def get_event_stats_data(
         self,
-        request,
-        organization,
-        get_event_stats,
-        top_events=0,
-        query_column="count()",
-        params=None,
-        query=None,
-        allow_partial_buckets=False,
-        zerofill_results=True,
-    ):
+        request: Request,
+        organization: Organization,
+        get_event_stats: Callable[
+            [Sequence[str], str, Dict[str, str], int, bool, Optional[timedelta]], SnubaTSResult
+        ],
+        top_events: int = 0,
+        query_column: str = "count()",
+        params: Optional[Dict[str, Any]] = None,
+        query: Optional[str] = None,
+        allow_partial_buckets: bool = False,
+        zerofill_results: bool = True,
+        comparison_delta: Optional[timedelta] = None,
+    ) -> Dict[str, Any]:
         with self.handle_query_errors():
             with sentry_sdk.start_span(
                 op="discover.endpoint", description="base.stats_query_creation"
@@ -287,11 +312,15 @@ def get_event_stats_data(
                 except InvalidSearchQuery:
                     sentry_sdk.set_tag("user.invalid_interval", request.GET.get("interval"))
                     date_range = params["end"] - params["start"]
-                    rollup = int(
-                        parse_stats_period(
-                            get_interval_from_range(date_range, False)
-                        ).total_seconds()
-                    )
+                    stats_period = parse_stats_period(get_interval_from_range(date_range, False))
+                    rollup = int(stats_period.total_seconds()) if stats_period is not None else 3600
+
+                if comparison_delta is not None:
+                    retention = quotas.get_event_retention(organization=organization)
+                    comparison_start = params["start"] - comparison_delta
+                    if retention and comparison_start < timezone.now() - timedelta(days=retention):
+                        raise ValidationError("Comparison period is outside your retention window")
+
                 # Backwards compatibility for incidents which uses the old
                 # column aliases as it straddles both versions of events/discover.
                 # We will need these aliases until discover2 flags are enabled for all
@@ -308,7 +337,9 @@ def get_event_stats_data(
 
                 query_columns = [column_map.get(column, column) for column in columns]
             with sentry_sdk.start_span(op="discover.endpoint", description="base.stats_query"):
-                result = get_event_stats(query_columns, query, params, rollup, zerofill_results)
+                result = get_event_stats(
+                    query_columns, query, params, rollup, zerofill_results, comparison_delta
+                )
 
         serializer = SnubaTSResultSerializer(organization, None, request.user)
 
@@ -359,13 +390,13 @@ def get_event_stats_data(
 
     def serialize_multiple_axis(
         self,
-        serializer,
-        event_result,
-        columns,
-        query_columns,
-        allow_partial_buckets,
-        zerofill_results=True,
-    ):
+        serializer: BaseSnubaSerializer,
+        event_result: SnubaTSResult,
+        columns: Sequence[str],
+        query_columns: Sequence[str],
+        allow_partial_buckets: bool,
+        zerofill_results: bool = True,
+    ) -> Dict[str, Any]:
         # Return with requested yAxis as the key
         result = {}
         equations = 0
@@ -387,10 +418,10 @@ def serialize_multiple_axis(
 
 
 class KeyTransactionBase(OrganizationEventsV2EndpointBase):
-    def has_feature(self, request, organization):
+    def has_feature(self, organization: Organization, request: Request) -> bool:
         return features.has("organizations:performance-view", organization, actor=request.user)
 
-    def get_project(self, request, organization):
+    def get_project(self, request: Request, organization: Organization) -> Project:
         projects = self.get_projects(request, organization)
 
         if len(projects) != 1: