Merge 4a4d6a4 into dcafe87

sentry-android-core/src/main/java/io/sentry/android/core/SendCachedEnvelopeIntegration.java

@@ -1,11 +1,13 @@
 package io.sentry.android.core;
 
+import io.sentry.DataCategory;
 import io.sentry.IConnectionStatusProvider;
 import io.sentry.IHub;
 import io.sentry.Integration;
 import io.sentry.SendCachedEnvelopeFireAndForgetIntegration;
 import io.sentry.SentryLevel;
 import io.sentry.SentryOptions;
+import io.sentry.transport.RateLimiter;
 import io.sentry.util.LazyEvaluator;
 import io.sentry.util.Objects;
 import java.io.Closeable;
@@ -81,6 +83,15 @@ private synchronized void sendCachedEnvelopes(
       return;
     }
 
+    // in case there's rate limiting active, skip processing
+    final @Nullable RateLimiter rateLimiter = hub.getRateLimiter();
+    if (rateLimiter != null && rateLimiter.isActiveForCategory(DataCategory.All)) {
+      options
+          .getLogger()
+          .log(SentryLevel.INFO, "SendCachedEnvelopeIntegration, rate limiting active.");
+      return;
+    }
+
     final SendCachedEnvelopeFireAndForgetIntegration.SendFireAndForget sender =
         factory.create(hub, options);
 

sentry-android-core/src/test/java/io/sentry/android/core/InternalSentrySdkTest.kt

@@ -23,6 +23,7 @@ import io.sentry.protocol.Mechanism
 import io.sentry.protocol.SentryId
 import io.sentry.protocol.User
 import io.sentry.transport.ITransport
+import io.sentry.transport.RateLimiter
 import org.junit.runner.RunWith
 import org.mockito.kotlin.mock
 import org.mockito.kotlin.whenever
@@ -63,6 +64,10 @@ class InternalSentrySdkTest {
                         override fun flush(timeoutMillis: Long) {
                             // no-op
                         }
+
+                        override fun getRateLimiter(): RateLimiter? {
+                            return null
+                        }
                     }
                 }
             }

sentry-apache-http-client-5/api/sentry-apache-http-client-5.api

@@ -2,6 +2,7 @@ public final class io/sentry/transport/apache/ApacheHttpClientTransport : io/sen
 	public fun <init> (Lio/sentry/SentryOptions;Lio/sentry/RequestDetails;Lorg/apache/hc/client5/http/impl/async/CloseableHttpAsyncClient;Lio/sentry/transport/RateLimiter;)V
 	public fun close ()V
 	public fun flush (J)V
+	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
 }
 

sentry-apache-http-client-5/src/main/java/io/sentry/transport/apache/ApacheHttpClientTransport.java

@@ -189,6 +189,11 @@ public void flush(long timeoutMillis) {
     }
   }
 
+  @Override
+  public @NotNull RateLimiter getRateLimiter() {
+    return rateLimiter;
+  }
+
   @Override
   public void close() throws IOException {
     options.getLogger().log(DEBUG, "Shutting down");

sentry/api/sentry.api

@@ -373,6 +373,7 @@ public final class io/sentry/Hub : io/sentry/IHub {
 	public fun getBaggage ()Lio/sentry/BaggageHeader;
 	public fun getLastEventId ()Lio/sentry/protocol/SentryId;
 	public fun getOptions ()Lio/sentry/SentryOptions;
+	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun getSpan ()Lio/sentry/ISpan;
 	public fun getTraceparent ()Lio/sentry/SentryTraceHeader;
 	public fun getTransaction ()Lio/sentry/ITransaction;
@@ -422,6 +423,7 @@ public final class io/sentry/HubAdapter : io/sentry/IHub {
 	public static fun getInstance ()Lio/sentry/HubAdapter;
 	public fun getLastEventId ()Lio/sentry/protocol/SentryId;
 	public fun getOptions ()Lio/sentry/SentryOptions;
+	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun getSpan ()Lio/sentry/ISpan;
 	public fun getTraceparent ()Lio/sentry/SentryTraceHeader;
 	public fun getTransaction ()Lio/sentry/ITransaction;
@@ -515,6 +517,7 @@ public abstract interface class io/sentry/IHub {
 	public abstract fun getBaggage ()Lio/sentry/BaggageHeader;
 	public abstract fun getLastEventId ()Lio/sentry/protocol/SentryId;
 	public abstract fun getOptions ()Lio/sentry/SentryOptions;
+	public abstract fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public abstract fun getSpan ()Lio/sentry/ISpan;
 	public abstract fun getTraceparent ()Lio/sentry/SentryTraceHeader;
 	public abstract fun getTransaction ()Lio/sentry/ITransaction;
@@ -608,6 +611,7 @@ public abstract interface class io/sentry/ISentryClient {
 	public abstract fun captureUserFeedback (Lio/sentry/UserFeedback;)V
 	public abstract fun close ()V
 	public abstract fun flush (J)V
+	public abstract fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public abstract fun isEnabled ()Z
 }
 
@@ -910,6 +914,7 @@ public final class io/sentry/NoOpHub : io/sentry/IHub {
 	public static fun getInstance ()Lio/sentry/NoOpHub;
 	public fun getLastEventId ()Lio/sentry/protocol/SentryId;
 	public fun getOptions ()Lio/sentry/SentryOptions;
+	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun getSpan ()Lio/sentry/ISpan;
 	public fun getTraceparent ()Lio/sentry/SentryTraceHeader;
 	public fun getTransaction ()Lio/sentry/ITransaction;
@@ -1516,6 +1521,7 @@ public final class io/sentry/SentryClient : io/sentry/ISentryClient {
 	public fun captureUserFeedback (Lio/sentry/UserFeedback;)V
 	public fun close ()V
 	public fun flush (J)V
+	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun isEnabled ()Z
 }
 
@@ -4138,6 +4144,7 @@ public final class io/sentry/transport/AsyncHttpTransport : io/sentry/transport/
 	public fun <init> (Lio/sentry/transport/QueuedThreadPoolExecutor;Lio/sentry/SentryOptions;Lio/sentry/transport/RateLimiter;Lio/sentry/transport/ITransportGate;Lio/sentry/transport/HttpConnection;)V
 	public fun close ()V
 	public fun flush (J)V
+	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
 }
 
@@ -4152,6 +4159,7 @@ public abstract interface class io/sentry/transport/ICurrentDateProvider {
 
 public abstract interface class io/sentry/transport/ITransport : java/io/Closeable {
 	public abstract fun flush (J)V
+	public abstract fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun send (Lio/sentry/SentryEnvelope;)V
 	public abstract fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
 }
@@ -4173,6 +4181,7 @@ public final class io/sentry/transport/NoOpTransport : io/sentry/transport/ITran
 	public fun close ()V
 	public fun flush (J)V
 	public static fun getInstance ()Lio/sentry/transport/NoOpTransport;
+	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
 }
 
@@ -4185,7 +4194,7 @@ public final class io/sentry/transport/RateLimiter {
 	public fun <init> (Lio/sentry/SentryOptions;)V
 	public fun <init> (Lio/sentry/transport/ICurrentDateProvider;Lio/sentry/SentryOptions;)V
 	public fun filter (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)Lio/sentry/SentryEnvelope;
-	public fun getRateLimitedUntilFor (Ljava/lang/String;)Ljava/util/Date;
+	public fun isActiveForCategory (Lio/sentry/DataCategory;)Z
 	public fun updateRetryAfterLimits (Ljava/lang/String;Ljava/lang/String;I)V
 }
 
@@ -4203,6 +4212,7 @@ public final class io/sentry/transport/StdoutTransport : io/sentry/transport/ITr
 	public fun <init> (Lio/sentry/ISerializer;)V
 	public fun close ()V
 	public fun flush (J)V
+	public fun getRateLimiter ()Lio/sentry/transport/RateLimiter;
 	public fun send (Lio/sentry/SentryEnvelope;Lio/sentry/Hint;)V
 }
 

sentry/src/main/java/io/sentry/DirectoryProcessor.java

@@ -3,23 +3,35 @@
 import static io.sentry.SentryLevel.ERROR;
 
 import io.sentry.hints.Cached;
+import io.sentry.hints.Enqueable;
 import io.sentry.hints.Flushable;
 import io.sentry.hints.Retryable;
 import io.sentry.hints.SubmissionResult;
+import io.sentry.transport.RateLimiter;
 import io.sentry.util.HintUtils;
 import java.io.File;
+import java.util.Queue;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 abstract class DirectoryProcessor {
 
+  private static final long ENVELOPE_PROCESS_DELAY = 100L;
+  private final @NotNull IHub hub;
   private final @NotNull ILogger logger;
   private final long flushTimeoutMillis;
+  private final Queue<String> processedEnvelopes;
 
-  DirectoryProcessor(final @NotNull ILogger logger, final long flushTimeoutMillis) {
+  DirectoryProcessor(
+      final @NotNull IHub hub, final @NotNull ILogger logger, final long flushTimeoutMillis) {
+    this.hub = hub;
     this.logger = logger;
     this.flushTimeoutMillis = flushTimeoutMillis;
+    this.processedEnvelopes =
+        SynchronizedQueue.synchronizedQueue(
+            new CircularFifoQueue<>(hub.getOptions().getMaxQueueSize()));
   }
 
   public void processDirectory(final @NotNull File directory) {
@@ -60,14 +72,37 @@ public void processDirectory(final @NotNull File directory) {
           continue;
         }
 
-        logger.log(SentryLevel.DEBUG, "Processing file: %s", file.getAbsolutePath());
+        final String filePath = file.getAbsolutePath();
+        // if envelope has already been submitted into the transport queue, we don't process it
+        // again
+        if (processedEnvelopes.contains(filePath)) {
+          logger.log(
+              SentryLevel.DEBUG,
+              "File '%s' has already been processed so it will not be processed again.",
+              filePath);
+          continue;
+        }
+
+        // in case there's rate limiting active, skip processing
+        final @Nullable RateLimiter rateLimiter = hub.getRateLimiter();
+        if (rateLimiter != null && rateLimiter.isActiveForCategory(DataCategory.All)) {
+          logger.log(SentryLevel.INFO, "DirectoryProcessor, rate limiting active.");
+          return;
+        }
+
+        logger.log(SentryLevel.DEBUG, "Processing file: %s", filePath);
 
         final SendCachedEnvelopeHint cachedHint =
-            new SendCachedEnvelopeHint(flushTimeoutMillis, logger);
+            new SendCachedEnvelopeHint(
+                flushTimeoutMillis, logger, () -> processedEnvelopes.add(filePath));
 
         final Hint hint = HintUtils.createWithTypeCheckHint(cachedHint);
-        hint.set(TypeCheckHint.SENTRY_CACHED_ENVELOPE_FILE_PATH, file.getAbsolutePath());
         processFile(file, hint);
+
+        // a short delay between processing envelopes to avoid bursting our server and hitting
+        // another rate limit https://develop.sentry.dev/sdk/features/#additional-capabilities
+        // InterruptedException will be handled by the outer try-catch
+        Thread.sleep(ENVELOPE_PROCESS_DELAY);
       }
     } catch (Throwable e) {
       logger.log(SentryLevel.ERROR, e, "Failed processing '%s'", directory.getAbsolutePath());
@@ -79,16 +114,21 @@ public void processDirectory(final @NotNull File directory) {
   protected abstract boolean isRelevantFileName(String fileName);
 
   private static final class SendCachedEnvelopeHint
-      implements Cached, Retryable, SubmissionResult, Flushable {
+      implements Cached, Retryable, SubmissionResult, Flushable, Enqueable {
     boolean retry = false;
     boolean succeeded = false;
 
     private final CountDownLatch latch;
     private final long flushTimeoutMillis;
     private final @NotNull ILogger logger;
+    private final @NotNull Runnable onEnqueued;
 
-    public SendCachedEnvelopeHint(final long flushTimeoutMillis, final @NotNull ILogger logger) {
+    public SendCachedEnvelopeHint(
+        final long flushTimeoutMillis,
+        final @NotNull ILogger logger,
+        final @NotNull Runnable onEnqueued) {
       this.flushTimeoutMillis = flushTimeoutMillis;
+      this.onEnqueued = onEnqueued;
       this.latch = new CountDownLatch(1);
       this.logger = logger;
     }
@@ -124,5 +164,10 @@ public void setResult(boolean succeeded) {
     public boolean isSuccess() {
       return succeeded;
     }
+
+    @Override
+    public void markEnqueued() {
+      onEnqueued.run();
+    }
   }
 }

sentry/src/main/java/io/sentry/EnvelopeSender.java

@@ -1,7 +1,6 @@
 package io.sentry;
 
 import io.sentry.cache.EnvelopeCache;
-import io.sentry.cache.IEnvelopeCache;
 import io.sentry.hints.Flushable;
 import io.sentry.hints.Retryable;
 import io.sentry.util.HintUtils;
@@ -21,19 +20,16 @@ public final class EnvelopeSender extends DirectoryProcessor implements IEnvelop
   private final @NotNull IHub hub;
   private final @NotNull ISerializer serializer;
   private final @NotNull ILogger logger;
-  private final @NotNull IEnvelopeCache envelopeCache;
 
   public EnvelopeSender(
       final @NotNull IHub hub,
       final @NotNull ISerializer serializer,
       final @NotNull ILogger logger,
-      final long flushTimeoutMillis,
-      final @NotNull IEnvelopeCache envelopeCache) {
-    super(logger, flushTimeoutMillis);
+      final long flushTimeoutMillis) {
+    super(hub, logger, flushTimeoutMillis);
     this.hub = Objects.requireNonNull(hub, "Hub is required.");
     this.serializer = Objects.requireNonNull(serializer, "Serializer is required.");
     this.logger = Objects.requireNonNull(logger, "Logger is required.");
-    this.envelopeCache = Objects.requireNonNull(envelopeCache, "IEnvelopeCache is required.");
   }
 
   @Override
@@ -57,14 +53,6 @@ protected void processFile(final @NotNull File file, final @NotNull Hint hint) {
       return;
     }
 
-    if (envelopeCache.containsFile(file)) {
-      logger.log(
-          SentryLevel.DEBUG,
-          "File '%s' is already in the cache so it will not be processed again.",
-          file.getAbsolutePath());
-      return;
-    }
-
     try (final InputStream is = new BufferedInputStream(new FileInputStream(file))) {
       SentryEnvelope envelope = serializer.deserializeEnvelope(is);
       if (envelope == null) {

sentry/src/main/java/io/sentry/Hub.java

@@ -7,6 +7,7 @@
 import io.sentry.protocol.SentryId;
 import io.sentry.protocol.SentryTransaction;
 import io.sentry.protocol.User;
+import io.sentry.transport.RateLimiter;
 import io.sentry.util.ExceptionUtils;
 import io.sentry.util.HintUtils;
 import io.sentry.util.Objects;
@@ -883,4 +884,10 @@ private Scope buildLocalScope(
 
     return null;
   }
+
+  @Override
+  public @Nullable RateLimiter getRateLimiter() {
+    final StackItem item = stack.peek();
+    return item.getClient().getRateLimiter();
+  }
 }

sentry/src/main/java/io/sentry/HubAdapter.java

@@ -3,6 +3,7 @@
 import io.sentry.protocol.SentryId;
 import io.sentry.protocol.SentryTransaction;
 import io.sentry.protocol.User;
+import io.sentry.transport.RateLimiter;
 import java.util.List;
 import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
@@ -262,4 +263,9 @@ public void reportFullyDisplayed() {
   public @Nullable BaggageHeader getBaggage() {
     return Sentry.getBaggage();
   }
+
+  @Override
+  public @Nullable RateLimiter getRateLimiter() {
+    return Sentry.getCurrentHub().getRateLimiter();
+  }
 }

sentry/src/main/java/io/sentry/IHub.java

@@ -3,6 +3,7 @@
 import io.sentry.protocol.SentryId;
 import io.sentry.protocol.SentryTransaction;
 import io.sentry.protocol.User;
+import io.sentry.transport.RateLimiter;
 import java.util.List;
 import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
@@ -633,4 +634,7 @@ TransactionContext continueTrace(
    */
   @Nullable
   BaggageHeader getBaggage();
+
+  @Nullable
+  RateLimiter getRateLimiter();
 }

sentry/src/main/java/io/sentry/ISentryClient.java

@@ -3,6 +3,7 @@
 import io.sentry.protocol.Message;
 import io.sentry.protocol.SentryId;
 import io.sentry.protocol.SentryTransaction;
+import io.sentry.transport.RateLimiter;
 import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -264,4 +265,8 @@ SentryId captureTransaction(
   default @NotNull SentryId captureTransaction(@NotNull SentryTransaction transaction) {
     return captureTransaction(transaction, null, null, null);
   }
+
+  @ApiStatus.Internal
+  @Nullable
+  RateLimiter getRateLimiter();
 }