ref: Remove sampling option, remove trimming from renormalization (#1…

…3563)

cont. #13358

src/sentry/models/event.py

@@ -22,7 +22,7 @@
 
 from semaphore.processing import StoreNormalizer
 
-from sentry import eventtypes
+from sentry import eventtypes, options
 from sentry.constants import EVENT_ORDERING_KEY
 from sentry.db.models import (
     BoundedBigIntegerField,
@@ -41,6 +41,17 @@
 from sentry.utils.strings import truncatechars
 
 
+def _should_disable_trim(event_id):
+    if not event_id:
+        return False
+
+    sample_rate = options.get('store.disable-trim-in-renormalization')
+    if sample_rate == 0:
+        return False
+
+    return int(md5(event_id).hexdigest(), 16) % (10 ** 8) <= (sample_rate * (10 ** 8))
+
+
 class EventDict(CanonicalKeyDict):
     """
     Creating an instance of this dictionary will send the event through basic
@@ -58,7 +69,10 @@ def __init__(self, data, skip_renormalization=False, **kwargs):
         )
 
         if not skip_renormalization and not is_renormalized:
-            normalizer = StoreNormalizer(is_renormalize=True)
+            normalizer = StoreNormalizer(
+                is_renormalize=True,
+                enable_trimming=not _should_disable_trim(data.get('event_id'))
+            )
             data = normalizer.normalize_event(dict(data))
 
         CanonicalKeyDict.__init__(self, data, **kwargs)

src/sentry/options/defaults.py

@@ -184,4 +184,5 @@
 
 
 # Normalization after processors
-register('store.normalize-after-processing', default=0.0)
+register('store.normalize-after-processing', default=0.0)  # unused
+register('store.disable-trim-in-renormalization', default=0.0)

src/sentry/tasks/store.py

@@ -10,7 +10,6 @@
 
 import logging
 from datetime import datetime
-import random
 import six
 
 from time import time
@@ -19,7 +18,7 @@
 
 from semaphore.processing import StoreNormalizer
 
-from sentry import features, reprocessing, options
+from sentry import features, reprocessing
 from sentry.constants import DEFAULT_STORE_NORMALIZER_ARGS
 from sentry.attachments import attachment_cache
 from sentry.cache import default_cache
@@ -44,11 +43,6 @@
 CRASH_REPORT_TYPES = ('event.minidump', )
 
 
-def _should_normalize_after_processing():
-    value = options.get('store.normalize-after-processing')
-    return value and random.random() < value
-
-
 class RetryProcessing(Exception):
     pass
 
@@ -273,17 +267,16 @@ def _do_process_event(cache_key, start_time, event_id, process_task,
         data = dict(data.items())
 
     if has_changed:
-        if _should_normalize_after_processing():
-            # Run some of normalization again such that we don't:
-            # - persist e.g. incredibly large stacktraces from minidumps
-            # - store event timestamps that are older than our retention window
-            #   (also happening with minidumps)
-            normalizer = StoreNormalizer(
-                remove_other=False,
-                is_renormalize=True,
-                **DEFAULT_STORE_NORMALIZER_ARGS
-            )
-            data = normalizer.normalize_event(dict(data))
+        # Run some of normalization again such that we don't:
+        # - persist e.g. incredibly large stacktraces from minidumps
+        # - store event timestamps that are older than our retention window
+        #   (also happening with minidumps)
+        normalizer = StoreNormalizer(
+            remove_other=False,
+            is_renormalize=True,
+            **DEFAULT_STORE_NORMALIZER_ARGS
+        )
+        data = normalizer.normalize_event(dict(data))
 
         issues = data.get('processing_issues')
 

src/sentry/utils/pytest/sentry.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+import pytest
 import mock
 import os
 
@@ -252,3 +253,8 @@ def pytest_runtest_teardown(item):
         model.objects.clear_local_cache()
 
     Hub.main.bind_client(None)
+
+
+@pytest.fixture(autouse=True)
+def _disable_renormalization_trim(monkeypatch):
+    monkeypatch.setattr('sentry.models.event._should_disable_trim', lambda _: True)