feat: New data category for indexed transactions [INGEST-1535]

CHANGELOG.md

@@ -32,6 +32,7 @@
 - Fix quota DataCategory::TransactionProcessed serialisation to match that of the CAPI. ([#1514](https://github.com/getsentry/relay/pull/1514))
 - Support checking quotas in the Redis rate limiter without incrementing them. ([#1519](https://github.com/getsentry/relay/pull/1519))
 - Update the internal service architecture for metrics aggregator service. ([#1508](https://github.com/getsentry/relay/pull/1508))
+- Add data category for indexed transactions. This will come to represent stored transactions, while the existing category will represent transaction metrics. ([#1535](https://github.com/getsentry/relay/pull/1535))
 
 ## 22.9.0
 

py/CHANGELOG.md

@@ -6,6 +6,7 @@
 - Add user-agent parsing to replays processor. ([#1420](https://github.com/getsentry/relay/pull/1420))
 - `convert_datascrubbing_config` will now return an error string when conversion fails on big regexes. ([#1474](https://github.com/getsentry/relay/pull/1474))
 - `relay_pii_strip_event` now treats any key containing `token` as a password. ([#1527](https://github.com/getsentry/relay/pull/1527))
+- Add data category for indexed transactions. This will come to represent stored transactions, while the existing category will represent transaction metrics. ([#1535](https://github.com/getsentry/relay/pull/1535))
 
 ## 0.8.13
 

relay-cabi/include/relay.h

@@ -3,7 +3,7 @@
 #ifndef RELAY_H_INCLUDED
 #define RELAY_H_INCLUDED
 
-/* Generated with cbindgen:0.23.0 */
+/* Generated with cbindgen:0.24.3 */
 
 /* Warning, this file is autogenerated. Do not modify this manually. */
 
@@ -49,9 +49,19 @@ enum RelayDataCategory {
    */
   RELAY_DATA_CATEGORY_REPLAY = 7,
   /**
-   * A transaction that was processed but not stored.
+   * DEPRECATED: A transaction for which metrics were extracted.
+   *
+   * This category is now obsolete because the `Transaction` variant will represent
+   * processed transactions from now on.
    */
   RELAY_DATA_CATEGORY_TRANSACTION_PROCESSED = 8,
+  /**
+   * Indexed transaction events.
+   *
+   * This is the category for transaction payloads that were accepted and stored in full. In
+   * contrast, `transaction` only guarantees that metrics have been accepted for the transaction.
+   */
+  RELAY_DATA_CATEGORY_TRANSACTION_INDEXED = 9,
   /**
    * Any other data category not known by this Relay.
    */

relay-common/src/constants.rs

@@ -113,8 +113,16 @@ pub enum DataCategory {
     Profile = 6,
     /// Session Replays
     Replay = 7,
-    /// A transaction that was processed but not stored.
+    /// DEPRECATED: A transaction for which metrics were extracted.
+    ///
+    /// This category is now obsolete because the `Transaction` variant will represent
+    /// processed transactions from now on.
     TransactionProcessed = 8,
+    /// Indexed transaction events.
+    ///
+    /// This is the category for transaction payloads that were accepted and stored in full. In
+    /// contrast, `transaction` only guarantees that metrics have been accepted for the transaction.
+    TransactionIndexed = 9,
     //
     // IMPORTANT: After adding a new entry to DataCategory, go to the `relay-cabi` subfolder and run
     // `make header` to regenerate the C-binding. This allows using the data category from Python.
@@ -139,6 +147,7 @@ impl DataCategory {
             "profile" => Self::Profile,
             "replay" => Self::Replay,
             "transaction_processed" => Self::TransactionProcessed,
+            "transaction_indexed" => Self::TransactionIndexed,
             _ => Self::Unknown,
         }
     }
@@ -156,6 +165,7 @@ impl DataCategory {
             Self::Profile => "profile",
             Self::Replay => "replay",
             Self::TransactionProcessed => "transaction_processed",
+            Self::TransactionIndexed => "transaction_indexed",
             Self::Unknown => "unknown",
         }
     }

relay-quotas/src/quota.rs

@@ -106,7 +106,8 @@ impl CategoryUnit {
             | DataCategory::Replay
             | DataCategory::Security
             | DataCategory::Profile
-            | DataCategory::TransactionProcessed => Some(Self::Count),
+            | DataCategory::TransactionProcessed
+            | DataCategory::TransactionIndexed => Some(Self::Count),
             DataCategory::Attachment => Some(Self::Bytes),
             DataCategory::Session => Some(Self::Batched),
             DataCategory::Unknown => None,

tests/integration/test_store.py

@@ -848,7 +848,8 @@ def counted_check_challenge(*args, **kwargs):
     assert counter[0] > 1
 
 
-def test_re_auth_failure(relay, mini_sentry):
+@pytest.mark.parametrize("run", range(10))
+def test_re_auth_failure(relay, mini_sentry, run):
     """
     Test that after a re-authentication failure, relay stops sending messages until is reauthenticated.
 
@@ -890,6 +891,9 @@ def counted_check_challenge(*args, **kwargs):
     auth_count_2 = counter[0]
     assert auth_count_1 < auth_count_2
 
+    # Give Relay some time to process the auth response and mark itself as not ready
+    sleep(0.1)
+
     # send a message, it should not come through while the authentication has failed
     relay.send_event(project_id, {"message": "123"})
     # sentry should have received nothing
@@ -906,6 +910,9 @@ def counted_check_challenge(*args, **kwargs):
     auth_count_3 = counter[0]
     assert auth_count_2 < auth_count_3
 
+    # Give Relay some time to process the auth response and mark itself as ready
+    sleep(0.1)
+
     # now we should be re-authenticated and we should have the event
 
     # sanity test that we got the event we sent