From 8095794019839c6b72b17c9278ce833123d024e9 Mon Sep 17 00:00:00 2001 From: Shashank Jarmale Date: Fri, 10 Oct 2025 14:08:32 -0700 Subject: [PATCH 1/4] Add double deletion of occurrences with EAP --- src/sentry/deletions/tasks/nodestore.py | 48 +++++++++++++- src/sentry/eventstream/eap_delete.py | 87 +++++++++++++++++++++++++ src/sentry/options/defaults.py | 8 +++ 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 src/sentry/eventstream/eap_delete.py diff --git a/src/sentry/deletions/tasks/nodestore.py b/src/sentry/deletions/tasks/nodestore.py index 76c87d860908f7..c408f2c52411b9 100644 --- a/src/sentry/deletions/tasks/nodestore.py +++ b/src/sentry/deletions/tasks/nodestore.py @@ -6,8 +6,9 @@ import sentry_sdk from snuba_sdk import DeleteQuery, Request -from sentry import eventstream, nodestore +from sentry import eventstream, nodestore, options from sentry.deletions.tasks.scheduled import MAX_RETRIES, logger +from sentry.eventstream.eap_delete import delete_groups_from_eap_rpc from sentry.exceptions import DeleteAborted from sentry.models.eventattachment import EventAttachment from sentry.models.userreport import UserReport @@ -205,6 +206,51 @@ def delete_events_from_eventstore( eventstream_state = eventstream.backend.start_delete_groups(project_id, group_ids) eventstream.backend.end_delete_groups(eventstream_state) + delete_events_from_eap(organization_id, project_id, group_ids, dataset) + + +def delete_events_from_eap( + organization_id: int, + project_id: int, + group_ids: Sequence[int], + dataset: Dataset, +) -> None: + eap_deletion_allowlist = options.get("eventstream.eap.deletion_enabled.project_allowlist") + if project_id not in eap_deletion_allowlist: + return + + try: + response = delete_groups_from_eap_rpc( + organization_id=organization_id, + project_id=project_id, + group_ids=group_ids, + referrer="deletions.group.eap", + ) + logger.info( + "eap.delete_groups.completed", + extra={ + "organization_id": organization_id, + "project_id": project_id, + "group_count": len(group_ids), + "matching_items_count": response.matching_items_count, + }, + ) + except Exception as e: + logger.exception( + "eap.delete_groups.failed", + extra={ + "organization_id": organization_id, + "project_id": project_id, + "group_ids": group_ids[:10], + "error": str(e), + }, + ) + metrics.incr( + "deletions.eap.failed", + tags={"dataset": dataset.value}, + sample_rate=1.0, + ) + def delete_events_from_eventstore_issue_platform( organization_id: int, project_id: int, group_ids: Sequence[int], times_seen_list: Sequence[int] diff --git a/src/sentry/eventstream/eap_delete.py b/src/sentry/eventstream/eap_delete.py new file mode 100644 index 00000000000000..6fc0b6789c6c90 --- /dev/null +++ b/src/sentry/eventstream/eap_delete.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import logging +from collections.abc import Sequence + +from sentry_protos.snuba.v1.endpoint_delete_trace_items_pb2 import ( + DeleteTraceItemsRequest, + DeleteTraceItemsResponse, +) +from sentry_protos.snuba.v1.request_common_pb2 import ( + TRACE_ITEM_TYPE_OCCURRENCE, + RequestMeta, + TraceItemFilterWithType, +) +from sentry_protos.snuba.v1.trace_item_attribute_pb2 import AttributeKey, AttributeValue, IntArray +from sentry_protos.snuba.v1.trace_item_filter_pb2 import ( + AndFilter, + ComparisonFilter, + TraceItemFilter, +) + +from sentry.utils import snuba_rpc + +logger = logging.getLogger(__name__) + + +def delete_groups_from_eap_rpc( + organization_id: int, + project_id: int, + group_ids: Sequence[int], + referrer: str = "deletions.group", +) -> DeleteTraceItemsResponse: + """ + Delete occurrences from EAP for the given group IDs. + """ + + if not group_ids: + raise ValueError("group_ids must not be empty") + + project_filter = _create_project_filter(project_id) + group_id_filter = _create_group_id_filter(list(group_ids)) + combined_filter = TraceItemFilter( + and_filter=AndFilter(filters=[project_filter, group_id_filter]) + ) + filter_with_type = TraceItemFilterWithType( + item_type=TRACE_ITEM_TYPE_OCCURRENCE, + filter=combined_filter, + ) + + request = DeleteTraceItemsRequest( + meta=RequestMeta( + organization_id=organization_id, + project_ids=[project_id], + referrer=referrer, + cogs_category="deletions", + ), + filters=[filter_with_type], + ) + response = snuba_rpc.rpc(request, DeleteTraceItemsResponse) + + return response + + +def _create_project_filter(project_id: int) -> TraceItemFilter: + return TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + type=AttributeKey.TYPE_INT, + name="sentry.project_id", + ), + op=ComparisonFilter.OP_EQUALS, + value=AttributeValue(val_int=project_id), + ) + ) + + +def _create_group_id_filter(group_ids: list[int]) -> TraceItemFilter: + return TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + type=AttributeKey.TYPE_INT, + name="sentry.group_id", + ), + op=ComparisonFilter.OP_IN, + value=AttributeValue(val_int_array=IntArray(values=group_ids)), + ) + ) diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 1c7ba24774c00c..b7ab3568fd410a 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -3518,3 +3518,11 @@ default=False, flags=FLAG_PRIORITIZE_DISK | FLAG_AUTOMATOR_MODIFIABLE, ) + +# Enables deletion from EAP for a set of projects +register( + "eventstream.eap.deletion_enabled.project_allowlist", + type=Sequence, + default=[], + flags=FLAG_ALLOW_EMPTY | FLAG_AUTOMATOR_MODIFIABLE, +) From 184b394e1e74ebef5fa3741d0e68742f0836432d Mon Sep 17 00:00:00 2001 From: Shashank Jarmale Date: Fri, 10 Oct 2025 14:08:52 -0700 Subject: [PATCH 2/4] Add tests for EAP double deletion logic --- tests/sentry/eventstream/test_eap_delete.py | 88 +++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/sentry/eventstream/test_eap_delete.py diff --git a/tests/sentry/eventstream/test_eap_delete.py b/tests/sentry/eventstream/test_eap_delete.py new file mode 100644 index 00000000000000..b3b0e2b1bd9be1 --- /dev/null +++ b/tests/sentry/eventstream/test_eap_delete.py @@ -0,0 +1,88 @@ +from unittest.mock import patch + +import pytest +from sentry_protos.snuba.v1.endpoint_delete_trace_items_pb2 import DeleteTraceItemsResponse +from sentry_protos.snuba.v1.request_common_pb2 import TRACE_ITEM_TYPE_OCCURRENCE, ResponseMeta + +from sentry.deletions.tasks.nodestore import delete_events_from_eap +from sentry.eventstream.eap_delete import delete_groups_from_eap_rpc +from sentry.snuba.dataset import Dataset +from sentry.testutils.cases import TestCase + + +class TestEAPDeletion(TestCase): + def setUp(self): + self.organization_id = 1 + self.project_id = 123 + self.group_ids = [1, 2, 3] + + @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") + def test_deletion_with_error_dataset(self, mock_rpc): + mock_rpc.return_value = DeleteTraceItemsResponse( + meta=ResponseMeta(), + matching_items_count=150, + ) + + with self.options( + {"eventstream.eap.deletion_enabled.project_allowlist": [self.project_id]} + ): + delete_events_from_eap( + self.organization_id, self.project_id, self.group_ids, Dataset.Events + ) + + assert mock_rpc.call_count == 1 + + request = mock_rpc.call_args[0][0] + assert request.meta.organization_id == self.organization_id + assert request.meta.project_ids == [self.project_id] + assert request.meta.referrer == "deletions.group.eap" + assert request.meta.cogs_category == "deletions" + + assert len(request.filters) == 1 + assert request.filters[0].item_type == TRACE_ITEM_TYPE_OCCURRENCE + + @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") + def test_multiple_group_ids(self, mock_rpc): + mock_rpc.return_value = DeleteTraceItemsResponse( + meta=ResponseMeta(), + matching_items_count=500, + ) + + many_group_ids = [10, 20, 30, 40, 50] + + with self.options( + {"eventstream.eap.deletion_enabled.project_allowlist": [self.project_id]} + ): + delete_events_from_eap( + self.organization_id, self.project_id, many_group_ids, Dataset.Events + ) + + request = mock_rpc.call_args[0][0] + group_filter = request.filters[0].filter.and_filter.filters[1] + assert list(group_filter.comparison_filter.value.val_int_array.values) == many_group_ids + + @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") + def test_project_not_in_allowlist_skips_deletion(self, mock_rpc): + with self.options({"eventstream.eap.deletion_enabled.project_allowlist": [456, 789]}): + delete_events_from_eap( + self.organization_id, self.project_id, self.group_ids, Dataset.Events + ) + + mock_rpc.assert_not_called() + + @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") + def test_empty_allowlist_skips_deletion(self, mock_rpc): + with self.options({"eventstream.eap.deletion_enabled.project_allowlist": []}): + delete_events_from_eap( + self.organization_id, self.project_id, self.group_ids, Dataset.Events + ) + + mock_rpc.assert_not_called() + + def test_empty_group_ids_raises_error(self): + with pytest.raises(ValueError, match="group_ids must not be empty"): + delete_groups_from_eap_rpc( + organization_id=self.organization_id, + project_id=self.project_id, + group_ids=[], + ) From 521a9637ddbadf217c706ec6e73f07bd3390e407 Mon Sep 17 00:00:00 2001 From: Shashank Jarmale Date: Tue, 14 Oct 2025 12:56:25 -0700 Subject: [PATCH 3/4] Change project allowlist to organization allowlist for EAP deletion --- src/sentry/deletions/tasks/nodestore.py | 4 ++-- src/sentry/options/defaults.py | 4 ++-- tests/sentry/eventstream/test_eap_delete.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/sentry/deletions/tasks/nodestore.py b/src/sentry/deletions/tasks/nodestore.py index c408f2c52411b9..b9e10cafe249aa 100644 --- a/src/sentry/deletions/tasks/nodestore.py +++ b/src/sentry/deletions/tasks/nodestore.py @@ -215,8 +215,8 @@ def delete_events_from_eap( group_ids: Sequence[int], dataset: Dataset, ) -> None: - eap_deletion_allowlist = options.get("eventstream.eap.deletion_enabled.project_allowlist") - if project_id not in eap_deletion_allowlist: + eap_deletion_allowlist = options.get("eventstream.eap.deletion_enabled.organization_allowlist") + if organization_id not in eap_deletion_allowlist: return try: diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 6441e226d8af08..2ca7b399cc71aa 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -3540,9 +3540,9 @@ flags=FLAG_PRIORITIZE_DISK | FLAG_AUTOMATOR_MODIFIABLE, ) -# Enables deletion from EAP for a set of projects +# The allowlist of organization IDs for which deletion from EAP is enabled. register( - "eventstream.eap.deletion_enabled.project_allowlist", + "eventstream.eap.deletion_enabled.organization_allowlist", type=Sequence, default=[], flags=FLAG_ALLOW_EMPTY | FLAG_AUTOMATOR_MODIFIABLE, diff --git a/tests/sentry/eventstream/test_eap_delete.py b/tests/sentry/eventstream/test_eap_delete.py index b3b0e2b1bd9be1..0da665dc2f4cc4 100644 --- a/tests/sentry/eventstream/test_eap_delete.py +++ b/tests/sentry/eventstream/test_eap_delete.py @@ -24,7 +24,7 @@ def test_deletion_with_error_dataset(self, mock_rpc): ) with self.options( - {"eventstream.eap.deletion_enabled.project_allowlist": [self.project_id]} + {"eventstream.eap.deletion_enabled.organization_allowlist": [self.organization_id]} ): delete_events_from_eap( self.organization_id, self.project_id, self.group_ids, Dataset.Events @@ -51,7 +51,7 @@ def test_multiple_group_ids(self, mock_rpc): many_group_ids = [10, 20, 30, 40, 50] with self.options( - {"eventstream.eap.deletion_enabled.project_allowlist": [self.project_id]} + {"eventstream.eap.deletion_enabled.organization_allowlist": [self.organization_id]} ): delete_events_from_eap( self.organization_id, self.project_id, many_group_ids, Dataset.Events @@ -62,8 +62,8 @@ def test_multiple_group_ids(self, mock_rpc): assert list(group_filter.comparison_filter.value.val_int_array.values) == many_group_ids @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") - def test_project_not_in_allowlist_skips_deletion(self, mock_rpc): - with self.options({"eventstream.eap.deletion_enabled.project_allowlist": [456, 789]}): + def test_organization_not_in_allowlist_skips_deletion(self, mock_rpc): + with self.options({"eventstream.eap.deletion_enabled.organization_allowlist": [456, 789]}): delete_events_from_eap( self.organization_id, self.project_id, self.group_ids, Dataset.Events ) @@ -72,7 +72,7 @@ def test_project_not_in_allowlist_skips_deletion(self, mock_rpc): @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") def test_empty_allowlist_skips_deletion(self, mock_rpc): - with self.options({"eventstream.eap.deletion_enabled.project_allowlist": []}): + with self.options({"eventstream.eap.deletion_enabled.organization_allowlist": []}): delete_events_from_eap( self.organization_id, self.project_id, self.group_ids, Dataset.Events ) From 5589ba537ce5e5bbca7cdd69deecc760eac76e31 Mon Sep 17 00:00:00 2001 From: Shashank Jarmale Date: Fri, 17 Oct 2025 12:33:52 -0700 Subject: [PATCH 4/4] Rename `eventstream/eap_delete.py` to `eventstream/eap.py` --- src/sentry/deletions/tasks/nodestore.py | 2 +- src/sentry/eventstream/{eap_delete.py => eap.py} | 0 .../eventstream/{test_eap_delete.py => test_eap.py} | 10 +++++----- 3 files changed, 6 insertions(+), 6 deletions(-) rename src/sentry/eventstream/{eap_delete.py => eap.py} (100%) rename tests/sentry/eventstream/{test_eap_delete.py => test_eap.py} (91%) diff --git a/src/sentry/deletions/tasks/nodestore.py b/src/sentry/deletions/tasks/nodestore.py index b9e10cafe249aa..2a79be32e7e7c3 100644 --- a/src/sentry/deletions/tasks/nodestore.py +++ b/src/sentry/deletions/tasks/nodestore.py @@ -8,7 +8,7 @@ from sentry import eventstream, nodestore, options from sentry.deletions.tasks.scheduled import MAX_RETRIES, logger -from sentry.eventstream.eap_delete import delete_groups_from_eap_rpc +from sentry.eventstream.eap import delete_groups_from_eap_rpc from sentry.exceptions import DeleteAborted from sentry.models.eventattachment import EventAttachment from sentry.models.userreport import UserReport diff --git a/src/sentry/eventstream/eap_delete.py b/src/sentry/eventstream/eap.py similarity index 100% rename from src/sentry/eventstream/eap_delete.py rename to src/sentry/eventstream/eap.py diff --git a/tests/sentry/eventstream/test_eap_delete.py b/tests/sentry/eventstream/test_eap.py similarity index 91% rename from tests/sentry/eventstream/test_eap_delete.py rename to tests/sentry/eventstream/test_eap.py index 0da665dc2f4cc4..f053c6849ca485 100644 --- a/tests/sentry/eventstream/test_eap_delete.py +++ b/tests/sentry/eventstream/test_eap.py @@ -5,7 +5,7 @@ from sentry_protos.snuba.v1.request_common_pb2 import TRACE_ITEM_TYPE_OCCURRENCE, ResponseMeta from sentry.deletions.tasks.nodestore import delete_events_from_eap -from sentry.eventstream.eap_delete import delete_groups_from_eap_rpc +from sentry.eventstream.eap import delete_groups_from_eap_rpc from sentry.snuba.dataset import Dataset from sentry.testutils.cases import TestCase @@ -16,7 +16,7 @@ def setUp(self): self.project_id = 123 self.group_ids = [1, 2, 3] - @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") + @patch("sentry.eventstream.eap.snuba_rpc.rpc") def test_deletion_with_error_dataset(self, mock_rpc): mock_rpc.return_value = DeleteTraceItemsResponse( meta=ResponseMeta(), @@ -41,7 +41,7 @@ def test_deletion_with_error_dataset(self, mock_rpc): assert len(request.filters) == 1 assert request.filters[0].item_type == TRACE_ITEM_TYPE_OCCURRENCE - @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") + @patch("sentry.eventstream.eap.snuba_rpc.rpc") def test_multiple_group_ids(self, mock_rpc): mock_rpc.return_value = DeleteTraceItemsResponse( meta=ResponseMeta(), @@ -61,7 +61,7 @@ def test_multiple_group_ids(self, mock_rpc): group_filter = request.filters[0].filter.and_filter.filters[1] assert list(group_filter.comparison_filter.value.val_int_array.values) == many_group_ids - @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") + @patch("sentry.eventstream.eap.snuba_rpc.rpc") def test_organization_not_in_allowlist_skips_deletion(self, mock_rpc): with self.options({"eventstream.eap.deletion_enabled.organization_allowlist": [456, 789]}): delete_events_from_eap( @@ -70,7 +70,7 @@ def test_organization_not_in_allowlist_skips_deletion(self, mock_rpc): mock_rpc.assert_not_called() - @patch("sentry.eventstream.eap_delete.snuba_rpc.rpc") + @patch("sentry.eventstream.eap.snuba_rpc.rpc") def test_empty_allowlist_skips_deletion(self, mock_rpc): with self.options({"eventstream.eap.deletion_enabled.organization_allowlist": []}): delete_events_from_eap(