test: improve debuggability of integration tests

embrace-android-delivery/src/main/kotlin/io/embrace/android/embracesdk/internal/delivery/debug/DeliveryTraceState.kt

@@ -131,6 +131,23 @@ internal sealed class DeliveryTraceState {
         }
     }
 
+    /**
+     * A HTTP request was started
+     */
+    class ServerReceivedRequest(private val endpoint: String) : DeliveryTraceState() {
+        override fun toString(): String = "ServerReceivedRequest, $endpoint"
+    }
+
+    /**
+     * A HTTP request was completed
+     */
+    class ServerCompletedRequest(
+        private val endpoint: String,
+        private val metadata: String
+    ) : DeliveryTraceState() {
+        override fun toString(): String = "ServerCompletedRequest, $endpoint $metadata"
+    }
+
     internal fun StoredTelemetryMetadata.toReportString(): String {
         return "$timestamp, $uuid, $payloadType, complete=$complete"
     }

embrace-android-delivery/src/main/kotlin/io/embrace/android/embracesdk/internal/delivery/debug/DeliveryTracer.kt

@@ -78,4 +78,12 @@ class DeliveryTracer {
     fun onPayloadResult(payload: StoredTelemetryMetadata, result: ExecutionResult) {
         events.add(DeliveryTraceState.PayloadResult(payload, result))
     }
+
+    fun onServerReceivedRequest(endpoint: String) {
+        events.add(DeliveryTraceState.ServerReceivedRequest(endpoint))
+    }
+
+    fun onServerCompletedRequest(endpoint: String, sessionId: String) {
+        events.add(DeliveryTraceState.ServerCompletedRequest(endpoint, sessionId))
+    }
 }

embrace-android-sdk/src/integrationTest/kotlin/io/embrace/android/embracesdk/testframework/IntegrationTestRule.kt

@@ -13,6 +13,7 @@ import io.embrace.android.embracesdk.fakes.config.FakeInstrumentedConfig
 import io.embrace.android.embracesdk.fakes.injection.FakeCoreModule
 import io.embrace.android.embracesdk.fakes.injection.FakeDeliveryModule
 import io.embrace.android.embracesdk.internal.config.remote.RemoteConfig
+import io.embrace.android.embracesdk.internal.delivery.debug.DeliveryTracer
 import io.embrace.android.embracesdk.internal.injection.CoreModule
 import io.embrace.android.embracesdk.internal.injection.DeliveryModule
 import io.embrace.android.embracesdk.internal.injection.EssentialServiceModule
@@ -121,9 +122,10 @@ internal class IntegrationTestRule(
     ) {
         setup = embraceSetupInterfaceSupplier()
         var apiServer: FakeApiServer? = null
+        val deliveryTracer = DeliveryTracer()
 
         if (setup.useMockWebServer) {
-            apiServer = FakeApiServer(serverResponseConfig)
+            apiServer = FakeApiServer(serverResponseConfig, deliveryTracer)
             val server: MockWebServer = MockWebServer().apply {
                 protocols = listOf(Protocol.HTTP_2, Protocol.HTTP_1_1)
                 dispatcher = apiServer
@@ -133,7 +135,7 @@ internal class IntegrationTestRule(
         }
 
         preSdkStart = EmbracePreSdkStartInterface(setup)
-        bootstrapper = setup.createBootstrapper(prepareConfig(instrumentedConfig))
+        bootstrapper = setup.createBootstrapper(prepareConfig(instrumentedConfig), deliveryTracer)
         action = EmbraceActionInterface(setup, bootstrapper)
         payloadAssertion = EmbracePayloadAssertionInterface(bootstrapper, apiServer)
         spanExporter = FilteredSpanExporter()
@@ -185,6 +187,7 @@ internal class IntegrationTestRule(
             setup.useMockWebServer -> instrumentedConfig.copy(
                 baseUrls = FakeBaseUrlConfig(configImpl = baseUrl, dataImpl = baseUrl)
             )
+
             else -> instrumentedConfig
         }
 

embrace-android-sdk/src/integrationTest/kotlin/io/embrace/android/embracesdk/testframework/actions/EmbracePayloadAssertionInterface.kt

@@ -86,7 +86,7 @@ internal class EmbracePayloadAssertionInterface(
                     "hashCodes" to envelope.data.logs?.map { it.hashCode() }?.joinToString { ", " }
                 )
             }
-            throwPayloadErrMsg(expectedSize, envelopes, exc)
+            throwPayloadErrMsg(expectedSize, envelopes.size, envelopes, exc)
         }
     }
 
@@ -161,14 +161,15 @@ internal class EmbracePayloadAssertionInterface(
         try {
             return retrievePayload(expectedSize, waitTimeMs, supplier)
         } catch (exc: TimeoutException) {
-            val sessions: List<Map<String, String?>> = supplier().map {
+            val envelopes = checkNotNull(apiServer).getSessionEnvelopes()
+            val sessions: List<Map<String, String?>> = envelopes.map {
                 mapOf(
                     "sessionId" to it.getSessionId(),
                     "cleanExit" to it.findSessionSpan().attributes?.findAttributeValue(embCleanExit.name),
                     "state" to it.findSessionSpan().attributes?.findAttributeValue(embState.name)
                 )
             }
-            throwPayloadErrMsg(expectedSize, sessions, exc)
+            throwPayloadErrMsg(expectedSize, envelopes.filter { it.findAppState() == appState }.size, sessions, exc)
         }
     }
 
@@ -379,12 +380,13 @@ internal class EmbracePayloadAssertionInterface(
 
     private fun throwPayloadErrMsg(
         expectedSize: Int,
+        observedSize: Int,
         envelopes: List<Map<String, String?>>,
         exc: TimeoutException,
     ): Nothing {
         throw IllegalStateException(
-            "Expected $expectedSize envelopes, but got ${envelopes.size}. " +
-                "Envelopes: $envelopes.\n${deliveryTracer.generateReport()}", exc
+            "Expected $expectedSize envelopes, but got $observedSize matching criteria. " +
+                "All received envelopes: $envelopes.\n${deliveryTracer.generateReport()}", exc
         )
     }
 }

embrace-android-sdk/src/integrationTest/kotlin/io/embrace/android/embracesdk/testframework/actions/EmbraceSetupInterface.kt

@@ -59,6 +59,7 @@ internal class EmbraceSetupInterface @JvmOverloads constructor(
 ) {
     fun createBootstrapper(
         instrumentedConfig: FakeInstrumentedConfig,
+        deliveryTracer: DeliveryTracer,
     ): ModuleInitBootstrapper = ModuleInitBootstrapper(
         initModule = overriddenInitModule.apply {
             this.instrumentedConfig = instrumentedConfig
@@ -102,7 +103,7 @@ internal class EmbraceSetupInterface @JvmOverloads constructor(
                 cacheStorageServiceProvider = cacheStorageServiceProvider,
                 requestExecutionServiceProvider = requestExecutionServiceProvider,
                 deliveryServiceProvider = deliveryServiceProvider,
-                deliveryTracer = DeliveryTracer()
+                deliveryTracer = deliveryTracer
             )
         },
         anrModuleSupplier = { _, _, _ -> fakeAnrModule },

embrace-android-sdk/src/integrationTest/kotlin/io/embrace/android/embracesdk/testframework/server/FakeApiServer.kt

@@ -1,13 +1,18 @@
 package io.embrace.android.embracesdk.testframework.server
 
+import io.embrace.android.embracesdk.assertions.getSessionId
 import io.embrace.android.embracesdk.fakes.TestPlatformSerializer
 import io.embrace.android.embracesdk.internal.TypeUtils
 import io.embrace.android.embracesdk.internal.config.remote.RemoteConfig
+import io.embrace.android.embracesdk.internal.delivery.debug.DeliveryTracer
 import io.embrace.android.embracesdk.internal.payload.Envelope
 import io.embrace.android.embracesdk.internal.payload.LogPayload
 import io.embrace.android.embracesdk.internal.payload.SessionPayload
+import io.embrace.android.embracesdk.internal.spans.findAttributeValue
 import io.embrace.android.embracesdk.internal.utils.threadLocal
-import java.util.concurrent.ConcurrentLinkedQueue
+import io.embrace.android.embracesdk.testframework.assertions.getLastLog
+import io.opentelemetry.semconv.incubating.LogIncubatingAttributes
+import java.util.concurrent.CopyOnWriteArrayList
 import java.util.zip.GZIPInputStream
 import okhttp3.mockwebserver.Dispatcher
 import okhttp3.mockwebserver.MockResponse
@@ -21,7 +26,10 @@ import org.junit.Assert.assertNotNull
 /**
  * Fake API server that is used to capture log/session requests made by the SDK in integration tests.
  */
-internal class FakeApiServer(private val remoteConfig: RemoteConfig) : Dispatcher() {
+internal class FakeApiServer(
+    private val remoteConfig: RemoteConfig,
+    private val deliveryTracer: DeliveryTracer
+) : Dispatcher() {
 
     private enum class Endpoint {
         LOGS,
@@ -30,9 +38,9 @@ internal class FakeApiServer(private val remoteConfig: RemoteConfig) : Dispatche
     }
 
     private val serializer by threadLocal { TestPlatformSerializer() }
-    private val sessionRequests = ConcurrentLinkedQueue<Envelope<SessionPayload>>()
-    private val logRequests = ConcurrentLinkedQueue<Envelope<LogPayload>>()
-    private val configRequests = ConcurrentLinkedQueue<String>()
+    private val sessionRequests = CopyOnWriteArrayList<Envelope<SessionPayload>>()
+    private val logRequests = CopyOnWriteArrayList<Envelope<LogPayload>>()
+    private val configRequests = CopyOnWriteArrayList<String>()
 
     /**
      * Returns a list of session envelopes in the order in which the server received them.
@@ -51,15 +59,16 @@ internal class FakeApiServer(private val remoteConfig: RemoteConfig) : Dispatche
     fun getConfigRequests(): List<String> = configRequests.toList()
 
     override fun dispatch(request: RecordedRequest): MockResponse {
-        return when (val endpoint = request.asEndpoint()) {
+        val endpoint = request.asEndpoint()
+        deliveryTracer.onServerReceivedRequest(endpoint.name)
+        return when (endpoint) {
             Endpoint.LOGS, Endpoint.SESSIONS -> handleEnvelopeRequest(request, endpoint)
 
             // IMPORTANT NOTE: this response is not used until the SDK next starts!
             Endpoint.CONFIG -> handleConfigRequest(request)
         }
     }
 
-    @Suppress("UNCHECKED_CAST")
     private fun handleEnvelopeRequest(
         request: RecordedRequest,
         endpoint: Endpoint,
@@ -68,13 +77,27 @@ internal class FakeApiServer(private val remoteConfig: RemoteConfig) : Dispatche
         validateHeaders(request.headers.toMultimap().mapValues { it.value.joinToString() })
 
         when (endpoint) {
-            Endpoint.SESSIONS -> sessionRequests.add(envelope as Envelope<SessionPayload>)
-            Endpoint.LOGS -> logRequests.add(envelope as Envelope<LogPayload>)
+            Endpoint.SESSIONS -> handleSessionRequest(endpoint, envelope)
+            Endpoint.LOGS -> handleLogRequest(endpoint, envelope)
             else -> error("Unsupported endpoint $endpoint")
         }
         return MockResponse().setResponseCode(200)
     }
 
+    @Suppress("UNCHECKED_CAST")
+    private fun handleSessionRequest(endpoint: Endpoint, envelope: Envelope<*>) {
+        val obj = envelope as Envelope<SessionPayload>
+        sessionRequests.add(obj)
+        deliveryTracer.onServerCompletedRequest(endpoint.name, obj.getSessionId())
+    }
+
+    @Suppress("UNCHECKED_CAST")
+    private fun handleLogRequest(endpoint: Endpoint, envelope: Envelope<*>) {
+        val obj = envelope as Envelope<LogPayload>
+        logRequests.add(obj)
+        deliveryTracer.onServerCompletedRequest(endpoint.name, obj.getLastLog().attributes?.findAttributeValue(LogIncubatingAttributes.LOG_RECORD_UID.key) ?: "")
+    }
+
     private fun handleConfigRequest(request: RecordedRequest): MockResponse {
         configRequests.add(request.requestUrl?.toUrl()?.query)
 
@@ -86,10 +109,12 @@ internal class FakeApiServer(private val remoteConfig: RemoteConfig) : Dispatche
             RemoteConfig::class.java,
             gzipSink.outputStream()
         )
-        return MockResponse()
+        val response = MockResponse()
             .setBody(configResponseBuffer)
             .addHeader("etag", "server_etag_value")
             .setResponseCode(200)
+        deliveryTracer.onServerCompletedRequest("config", "")
+        return response
     }
 
     private fun validateHeaders(headers: Map<String, String>) {