From 3dca813f1ba1c623a11a17e4755dc5b217474074 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 16 Feb 2024 12:12:32 -0800 Subject: [PATCH 01/21] Update EH to the latest otel spec and rework metrics --- .../checkstyle/checkstyle-suppressions.xml | 5 +- .../handler/SessionHandler.java | 6 + .../CHANGELOG.md | 2 + .../blob/BlobCheckpointStore.java | 24 +- .../checkpointstore/blob/MetricsHelper.java | 197 ------ .../blob/MetricsHelperTests.java | 317 --------- .../blob/TestMeterProvider.java | 26 - .../Chart.yaml | 4 +- .../workbooks/details.json | 41 +- .../azure-messaging-eventhubs/CHANGELOG.md | 8 + .../EventHubBufferedPartitionProducer.java | 2 +- .../eventhubs/EventHubClientBuilder.java | 9 +- .../EventHubConsumerAsyncClient.java | 6 +- .../eventhubs/EventHubConsumerClient.java | 27 +- .../EventHubProducerAsyncClient.java | 8 +- .../EventHubsProducerInstrumentation.java | 66 +- .../eventhubs/EventProcessorClient.java | 18 +- .../EventProcessorClientBuilder.java | 2 +- .../eventhubs/PartitionPumpManager.java | 31 +- .../eventhubs/SynchronousReceiver.java | 140 ++++ .../AmqpReceiveLinkProcessor.java | 9 +- .../implementation/ClientConstants.java | 1 - .../EventHubsConsumerInstrumentation.java | 123 +++- .../EventHubsMetricsProvider.java | 185 +++-- .../instrumentation/EventHubsTracer.java | 250 ++++--- .../instrumentation/InstrumentationScope.java | 110 +++ .../instrumentation/InstrumentationUtils.java | 106 +++ .../InstrumentedCheckpointStore.java | 68 ++ .../instrumentation/OperationName.java | 27 + .../src/main/java/module-info.java | 2 +- .../EventHubConsumerAsyncClientTest.java | 62 +- .../EventHubProducerAsyncClientTest.java | 242 ++++--- .../eventhubs/EventHubProducerClientTest.java | 55 +- ...EventHubsProducerInstrumentationTests.java | 220 ++++++ ...EventProcessorClientErrorHandlingTest.java | 8 +- .../eventhubs/EventProcessorClientTest.java | 586 +++++++++++----- .../eventhubs/IntegrationTestBase.java | 5 +- .../PartitionBasedLoadBalancerTest.java | 16 +- .../eventhubs/PartitionPumpManagerTest.java | 38 +- .../eventhubs/TestSpanProcessor.java | 117 ++++ .../azure/messaging/eventhubs/TestUtils.java | 70 ++ .../eventhubs/TracingIntegrationTests.java | 323 ++++----- ...EventHubsConsumerInstrumentationTests.java | 649 ++++++++++++++++++ .../instrumentation/EventHubsTracerTests.java | 33 +- .../InstrumentationScopeTests.java | 163 +++++ 45 files changed, 3022 insertions(+), 1385 deletions(-) delete mode 100644 sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/main/java/com/azure/messaging/eventhubs/checkpointstore/blob/MetricsHelper.java delete mode 100644 sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/test/java/com/azure/messaging/eventhubs/checkpointstore/blob/MetricsHelperTests.java delete mode 100644 sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/test/java/com/azure/messaging/eventhubs/checkpointstore/blob/TestMeterProvider.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedCheckpointStore.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/OperationName.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentationTests.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentationTests.java create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScopeTests.java diff --git a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml index 2650b791f334f..289f03bd3adbc 100644 --- a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml +++ b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml @@ -81,9 +81,8 @@ - - - + + blobClients = new ConcurrentHashMap<>(); /** @@ -93,7 +89,6 @@ public BlobCheckpointStore(BlobContainerAsyncClient blobContainerAsyncClient) { */ public BlobCheckpointStore(BlobContainerAsyncClient blobContainerAsyncClient, ClientOptions options) { this.blobContainerAsyncClient = blobContainerAsyncClient; - this.metricsHelper = new MetricsHelper(options == null ? null : options.getMetricsOptions(), MeterProvider.getDefaultProvider()); } /** @@ -267,7 +262,7 @@ public Mono updateCheckpoint(Checkpoint checkpoint) { metadata.put(OFFSET, offset); BlobAsyncClient blobAsyncClient = blobClients.get(blobName); - Mono response = blobAsyncClient.exists().flatMap(exists -> { + return blobAsyncClient.exists().flatMap(exists -> { if (exists) { return blobAsyncClient.setMetadata(metadata); } else { @@ -275,22 +270,6 @@ public Mono updateCheckpoint(Checkpoint checkpoint) { metadata, null, null, null).then(); } }); - return reportMetrics(response, checkpoint, blobName); - } - - private Mono reportMetrics(Mono checkpointMono, Checkpoint checkpoint, String blobName) { - AtomicReference startTime = metricsHelper.isCheckpointDurationEnabled() ? new AtomicReference<>() : null; - return checkpointMono - .doOnEach(signal -> { - if (signal.isOnComplete() || signal.isOnError()) { - metricsHelper.reportCheckpoint(checkpoint, blobName, !signal.hasError(), startTime != null ? startTime.get() : null); - } - }) - .doOnSubscribe(ignored -> { - if (startTime != null) { - startTime.set(Instant.now()); - } - }); } private String getBlobPrefix(String fullyQualifiedNamespace, String eventHubName, String consumerGroupName, @@ -348,5 +327,4 @@ private Mono convertToPartitionOwnership(BlobItem blobItem) return Mono.empty(); } - } diff --git a/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/main/java/com/azure/messaging/eventhubs/checkpointstore/blob/MetricsHelper.java b/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/main/java/com/azure/messaging/eventhubs/checkpointstore/blob/MetricsHelper.java deleted file mode 100644 index cd5bd44eeb8d2..0000000000000 --- a/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/main/java/com/azure/messaging/eventhubs/checkpointstore/blob/MetricsHelper.java +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.messaging.eventhubs.checkpointstore.blob; - -import com.azure.core.util.Context; -import com.azure.core.util.CoreUtils; -import com.azure.core.util.MetricsOptions; -import com.azure.core.util.TelemetryAttributes; -import com.azure.core.util.logging.ClientLogger; -import com.azure.core.util.metrics.DoubleHistogram; -import com.azure.core.util.metrics.LongGauge; -import com.azure.core.util.metrics.Meter; -import com.azure.core.util.metrics.MeterProvider; -import com.azure.messaging.eventhubs.models.Checkpoint; - -import java.time.Duration; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; - -final class MetricsHelper { - private static final ClientLogger LOGGER = new ClientLogger(MetricsHelper.class); - - // Make sure attribute names are consistent across AMQP Core, EventHubs, ServiceBus when applicable - // and mapped correctly in OTel Metrics https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/core/azure-core-metrics-opentelemetry/src/main/java/com/azure/core/metrics/opentelemetry/OpenTelemetryAttributes.java - private static final String ENTITY_NAME_KEY = "entityName"; - private static final String HOSTNAME_KEY = "hostName"; - private static final String PARTITION_ID_KEY = "partitionId"; - private static final String CONSUMER_GROUP_KEY = "consumerGroup"; - private static final String STATUS_KEY = "status"; - - // since checkpoint store is stateless it might be used for endless number of eventhubs. - // we'll have as many subscriptions as there are combinations of fqdn, eventhub name, partitionId and consumer group. - // In the unlikely case it's shared across a lot of EH client instances, metrics would be too costly - // and unhelpful. So, let's just set a hard limit on number of subscriptions. - private static final int MAX_ATTRIBUTES_SETS = 100; - - private static final String PROPERTIES_FILE = "azure-messaging-eventhubs-checkpointstore-blob.properties"; - private static final String NAME_KEY = "name"; - private static final String VERSION_KEY = "version"; - private static final String LIBRARY_NAME; - private static final String LIBRARY_VERSION; - private static final String UNKNOWN = "UNKNOWN"; - - static { - final Map properties = CoreUtils.getProperties(PROPERTIES_FILE); - LIBRARY_NAME = properties.getOrDefault(NAME_KEY, UNKNOWN); - LIBRARY_VERSION = properties.getOrDefault(VERSION_KEY, UNKNOWN); - } - - private final ConcurrentHashMap common = new ConcurrentHashMap<>(); - private final ConcurrentHashMap checkpointFailure = new ConcurrentHashMap<>(); - private final ConcurrentHashMap checkpointSuccess = new ConcurrentHashMap<>(); - private final ConcurrentHashMap seqNoSubscriptions = new ConcurrentHashMap<>(); - - private volatile boolean maxCapacityReached = false; - - private final Meter meter; - private final LongGauge lastSequenceNumber; - private final DoubleHistogram checkpointDuration; - private final boolean isEnabled; - - MetricsHelper(MetricsOptions metricsOptions, MeterProvider meterProvider) { - if (areMetricsEnabled(metricsOptions)) { - this.meter = meterProvider.createMeter(LIBRARY_NAME, LIBRARY_VERSION, metricsOptions); - this.isEnabled = this.meter.isEnabled(); - } else { - this.meter = null; - this.isEnabled = false; - } - - if (isEnabled) { - this.lastSequenceNumber = this.meter.createLongGauge("messaging.eventhubs.checkpoint.sequence_number", "Last successfully checkpointed sequence number.", "seqNo"); - this.checkpointDuration = this.meter.createDoubleHistogram("messaging.eventhubs.checkpoint.duration", "Duration of checkpoint call.", "ms"); - } else { - this.lastSequenceNumber = null; - this.checkpointDuration = null; - } - } - - boolean isCheckpointDurationEnabled() { - return isEnabled && checkpointDuration.isEnabled(); - } - - void reportCheckpoint(Checkpoint checkpoint, String attributesId, boolean success, Instant startTime) { - if (!isEnabled || !(lastSequenceNumber.isEnabled() && checkpointDuration.isEnabled())) { - return; - } - - if (!maxCapacityReached && (seqNoSubscriptions.size() >= MAX_ATTRIBUTES_SETS || common.size() >= MAX_ATTRIBUTES_SETS)) { - LOGGER.error("Too many attribute combinations are reported for checkpoint metrics, ignoring any new dimensions."); - maxCapacityReached = true; - } - - if (lastSequenceNumber.isEnabled() && success) { - updateCurrentValue(attributesId, checkpoint); - } - - if (checkpointDuration.isEnabled()) { - TelemetryAttributes attributes; - if (success) { - attributes = getOrCreate(checkpointSuccess, attributesId, checkpoint, null); - } else { - attributes = getOrCreate(checkpointFailure, attributesId, checkpoint, "error"); - } - - if (attributes != null) { - if (checkpointDuration.isEnabled()) { - checkpointDuration.record(Duration.between(startTime, Instant.now()).toMillis(), attributes, Context.NONE); - } - } - } - } - - private TelemetryAttributes getOrCreate(ConcurrentHashMap source, String attributesId, Checkpoint checkpoint, String status) { - if (maxCapacityReached) { - return source.get(attributesId); - } - - return source.computeIfAbsent(attributesId, i -> meter.createAttributes(createAttributes(checkpoint, status))); - } - - private Map createAttributes(Checkpoint checkpoint, String status) { - Map attributesMap = new HashMap<>(5); - attributesMap.put(HOSTNAME_KEY, checkpoint.getFullyQualifiedNamespace()); - attributesMap.put(ENTITY_NAME_KEY, checkpoint.getEventHubName()); - attributesMap.put(PARTITION_ID_KEY, checkpoint.getPartitionId()); - attributesMap.put(CONSUMER_GROUP_KEY, checkpoint.getConsumerGroup()); - if (status != null) { - attributesMap.put(STATUS_KEY, status); - } - - return attributesMap; - } - - private void updateCurrentValue(String attributesId, Checkpoint checkpoint) { - if (checkpoint.getSequenceNumber() == null) { - return; - } - - final CurrentValue valueSupplier; - if (maxCapacityReached) { - valueSupplier = seqNoSubscriptions.get(attributesId); - if (valueSupplier == null) { - return; - } - } else { - TelemetryAttributes attributes = getOrCreate(common, attributesId, checkpoint, null); - if (attributes == null) { - return; - } - - valueSupplier = seqNoSubscriptions.computeIfAbsent(attributesId, a -> { - AtomicReference lastSeqNo = new AtomicReference<>(); - return new CurrentValue(lastSequenceNumber.registerCallback(() -> lastSeqNo.get(), attributes), lastSeqNo); - }); - } - - valueSupplier.set(checkpoint.getSequenceNumber()); - } - - private static boolean areMetricsEnabled(MetricsOptions options) { - if (options == null || options.isEnabled()) { - return true; - } - - return false; - } - - private static class CurrentValue { - private final AtomicReference lastSeqNo; - private final AutoCloseable subscription; - - CurrentValue(AutoCloseable subscription, AtomicReference lastSeqNo) { - this.subscription = subscription; - this.lastSeqNo = lastSeqNo; - } - - void set(long value) { - lastSeqNo.set(value); - } - - void close() { - if (subscription != null) { - try { - subscription.close(); - } catch (Exception e) { - // should never happen - throw LOGGER.logThrowableAsWarning(new RuntimeException(e)); - } - } - } - } -} diff --git a/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/test/java/com/azure/messaging/eventhubs/checkpointstore/blob/MetricsHelperTests.java b/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/test/java/com/azure/messaging/eventhubs/checkpointstore/blob/MetricsHelperTests.java deleted file mode 100644 index 55618cddf46cc..0000000000000 --- a/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/test/java/com/azure/messaging/eventhubs/checkpointstore/blob/MetricsHelperTests.java +++ /dev/null @@ -1,317 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.messaging.eventhubs.checkpointstore.blob; - -import com.azure.core.test.utils.metrics.TestGauge; -import com.azure.core.test.utils.metrics.TestHistogram; -import com.azure.core.test.utils.metrics.TestMeasurement; -import com.azure.core.test.utils.metrics.TestMeter; -import com.azure.core.util.MetricsOptions; -import com.azure.core.util.metrics.Meter; -import com.azure.messaging.eventhubs.models.Checkpoint; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.junit.jupiter.api.parallel.Isolated; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@Execution(ExecutionMode.SAME_THREAD) -@Isolated -public class MetricsHelperTests { - private static final int MAX_ATTRIBUTES_SETS = 100; - - @Test - public void testUpdateDisabledMetrics() { - Checkpoint checkpoint = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh") - .setConsumerGroup("cg") - .setPartitionId("0") - .setSequenceNumber(2L) - .setOffset(100L); - - Meter meter = mock(Meter.class); - when(meter.isEnabled()).thenReturn(false); - - TestMeterProvider testProvider = new TestMeterProvider((lib, ver, opts) -> { - assertEquals("azure-messaging-eventhubs-checkpointstore-blob", lib); - assertNotNull(ver); - return meter; - }); - - - MetricsHelper helper = new MetricsHelper(new MetricsOptions(), testProvider); - helper.reportCheckpoint(checkpoint, "ns/eh/ch/0", true, null); - - verify(meter, atLeastOnce()).isEnabled(); - verify(meter, never()).createAttributes(anyMap()); - verify(meter, never()).createLongGauge(any(), any(), any()); - verify(meter, never()).createDoubleHistogram(any(), any(), any()); - } - - @Test - public void testUpdateDisabledMetricsViaOptions() { - Checkpoint checkpoint = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh") - .setConsumerGroup("cg") - .setPartitionId("0") - .setSequenceNumber(2L) - .setOffset(100L); - - Meter meter = mock(Meter.class); - TestMeterProvider testProvider = new TestMeterProvider((lib, ver, opts) -> meter); - MetricsHelper helper = new MetricsHelper(new MetricsOptions().setEnabled(false), testProvider); - helper.reportCheckpoint(checkpoint, "ns/eh/cg/0", true, null); - - verify(meter, never()).createAttributes(anyMap()); - verify(meter, never()).createLongGauge(any(), any(), any()); - verify(meter, never()).createDoubleHistogram(any(), any(), any()); - } - - @Test - public void testUpdateEnabledMetrics() { - Checkpoint checkpoint = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh") - .setConsumerGroup("cg") - .setPartitionId("0") - .setSequenceNumber(2L) - .setOffset(100L); - - TestMeter meter = new TestMeter(); - TestMeterProvider testProvider = new TestMeterProvider((lib, ver, opts) -> { - assertEquals("azure-messaging-eventhubs-checkpointstore-blob", lib); - assertNotNull(ver); - return meter; - }); - - MetricsHelper helper = new MetricsHelper(new MetricsOptions(), testProvider); - Instant startTime = Instant.now().minus(10, ChronoUnit.SECONDS); - helper.reportCheckpoint(checkpoint, "ns/eh/cg/0", true, startTime); - - assertTrue(meter.getGauges().containsKey("messaging.eventhubs.checkpoint.sequence_number")); - TestGauge seqNo = meter.getGauges().get("messaging.eventhubs.checkpoint.sequence_number"); - assertEquals(1, seqNo.getSubscriptions().size()); - TestGauge.Subscription subs = seqNo.getSubscriptions().get(0); - - assertEquals(0, subs.getMeasurements().size()); - subs.measure(); - - TestMeasurement seqNoMeasurement = subs.getMeasurements().get(0); - assertEquals(2L, seqNoMeasurement.getValue()); - assertAttributes(checkpoint, null, seqNoMeasurement.getAttributes()); - - assertTrue(meter.getHistograms().containsKey("messaging.eventhubs.checkpoint.duration")); - TestHistogram checkpointDuration = meter.getHistograms().get("messaging.eventhubs.checkpoint.duration"); - assertEquals(1, checkpointDuration.getMeasurements().size()); - TestMeasurement durationMeasurements = checkpointDuration.getMeasurements().get(0); - assertEquals(10000d, durationMeasurements.getValue(), 1000d); - assertAttributes(checkpoint, null, durationMeasurements.getAttributes()); - } - - @Test - public void testUpdateEnabledMetricsFailure() { - Checkpoint checkpoint = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh") - .setConsumerGroup("cg") - .setPartitionId("0") - .setSequenceNumber(2L) - .setOffset(100L); - - TestMeter meter = new TestMeter(); - Instant startTime = Instant.now().minus(1, ChronoUnit.SECONDS); - MetricsHelper helper = new MetricsHelper(new MetricsOptions(), new TestMeterProvider((lib, ver, opts) -> meter)); - helper.reportCheckpoint(checkpoint, "ns/eh/cg/0", false, startTime); - - // sequence number is only reported for successful checkpoints - assertEquals(0, meter.getGauges().get("messaging.eventhubs.checkpoint.sequence_number").getSubscriptions().size()); - - TestHistogram checkpointDuration = meter.getHistograms().get("messaging.eventhubs.checkpoint.duration"); - TestMeasurement durationMeasurements = checkpointDuration.getMeasurements().get(0); - assertEquals(1000d, durationMeasurements.getValue(), 1000d); - assertAttributes(checkpoint, "error", durationMeasurements.getAttributes()); - } - - @Test - public void testUpdateEnabledMetricsNullSeqNo() { - Checkpoint checkpoint = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh") - .setConsumerGroup("cg") - .setPartitionId("0") - .setOffset(100L); - - TestMeter meter = new TestMeter(); - Instant startTime = Instant.now(); - MetricsHelper helper = new MetricsHelper(new MetricsOptions(), new TestMeterProvider((lib, ver, opts) -> meter)); - helper.reportCheckpoint(checkpoint, "ns/eh/cg/0", true, startTime); - - assertEquals(0, meter.getGauges().get("messaging.eventhubs.checkpoint.sequence_number").getSubscriptions().size()); - - TestHistogram checkpointDuration = meter.getHistograms().get("messaging.eventhubs.checkpoint.duration"); - TestMeasurement durationMeasurements = checkpointDuration.getMeasurements().get(0); - assertEquals(0d, durationMeasurements.getValue(), 1000d); - assertAttributes(checkpoint, null, durationMeasurements.getAttributes()); - } - - @Test - public void testUpdateEnabledMetricsTooManyAttributes() { - TestMeter meter = new TestMeter(); - List checkpoints = IntStream.range(0, MAX_ATTRIBUTES_SETS + 10) - .mapToObj(n -> new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh") - .setConsumerGroup("cg") - .setPartitionId(String.valueOf(n)) - .setSequenceNumber((long) n) - .setOffset(100L)) - .collect(Collectors.toList()); - - MetricsHelper helper = new MetricsHelper(new MetricsOptions(), new TestMeterProvider((lib, ver, opts) -> meter)); - checkpoints.forEach(ch -> helper.reportCheckpoint(ch, "ns/eh/cg/" + ch.getPartitionId(), true, Instant.now())); - - List subscriptions = meter.getGauges().get("messaging.eventhubs.checkpoint.sequence_number").getSubscriptions(); - assertEquals(MAX_ATTRIBUTES_SETS, subscriptions.size()); - subscriptions.forEach(subs -> subs.measure()); - - final int[] i = {0}; - subscriptions.forEach(subs -> { - assertEquals(1, subs.getMeasurements().size()); - TestMeasurement seqNoMeasurement = subs.getMeasurements().get(0); - assertEquals(i[0], seqNoMeasurement.getValue()); - assertAttributes(checkpoints.get(i[0]), null, seqNoMeasurement.getAttributes()); - i[0]++; - }); - - TestHistogram durationHistogram = meter.getHistograms().get("messaging.eventhubs.checkpoint.duration"); - assertEquals(MAX_ATTRIBUTES_SETS, durationHistogram.getMeasurements().size()); - - final int[] k = {0}; - durationHistogram.getMeasurements().forEach(m -> { - assertEquals(0d, m.getValue(), 1000d); - assertAttributes(checkpoints.get(k[0]), null, m.getAttributes()); - k[0]++; - }); - } - - @Test - public void testUpdateEnabledMetricsMultipleMeasurements() { - Checkpoint checkpoint1 = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh") - .setConsumerGroup("cg") - .setPartitionId("0") - .setSequenceNumber(2L) - .setOffset(100L); - - Checkpoint checkpoint2 = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh") - .setConsumerGroup("cg") - .setPartitionId("0") - .setSequenceNumber(42L) - .setOffset(100L); - - TestMeter meter = new TestMeter(); - MetricsHelper helper = new MetricsHelper(new MetricsOptions(), new TestMeterProvider((lib, ver, opts) -> meter)); - helper.reportCheckpoint(checkpoint1, "ns/eh/cg/0", true, Instant.now().minus(10, ChronoUnit.SECONDS)); - helper.reportCheckpoint(checkpoint2, "ns/eh/cg/0", true, Instant.now()); - - TestGauge seqNo = meter.getGauges().get("messaging.eventhubs.checkpoint.sequence_number"); - TestGauge.Subscription subs = seqNo.getSubscriptions().get(0); - subs.measure(); - - TestMeasurement seqNoMeasurement = subs.getMeasurements().get(0); - assertEquals(42L, seqNoMeasurement.getValue()); - - TestHistogram duration = meter.getHistograms().get("messaging.eventhubs.checkpoint.duration"); - assertEquals(2, duration.getMeasurements().size()); - - assertEquals(10000d, duration.getMeasurements().get(0).getValue(), 1000d); - assertEquals(0d, duration.getMeasurements().get(1).getValue(), 1000d); - assertAttributes(checkpoint2, null, duration.getMeasurements().get(1).getAttributes()); - } - - @Test - public void testUpdateEnabledMetricsMultipleHubs() { - Checkpoint checkpoint1 = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh1") - .setConsumerGroup("cg") - .setPartitionId("0") - .setSequenceNumber(2L) - .setOffset(100L); - - Checkpoint checkpoint2 = new Checkpoint() - .setFullyQualifiedNamespace("ns") - .setEventHubName("eh2") - .setConsumerGroup("cg") - .setPartitionId("0") - .setSequenceNumber(42L) - .setOffset(100L); - - TestMeter meter = new TestMeter(); - MetricsHelper helper = new MetricsHelper(new MetricsOptions(), new TestMeterProvider((lib, ver, opts) -> meter)); - - helper.reportCheckpoint(checkpoint1, "ns/eh1/cg/0", true, Instant.now()); - helper.reportCheckpoint(checkpoint2, "ns/eh2/cg/0", true, Instant.now()); - - TestGauge seqNo = meter.getGauges().get("messaging.eventhubs.checkpoint.sequence_number"); - assertEquals(2, seqNo.getSubscriptions().size()); - TestGauge.Subscription subs1 = seqNo.getSubscriptions().get(0); - TestGauge.Subscription subs2 = seqNo.getSubscriptions().get(1); - subs1.measure(); - subs2.measure(); - - TestMeasurement seqNoMeasurement1 = subs1.getMeasurements().get(0); - assertEquals(2L, seqNoMeasurement1.getValue()); - assertAttributes(checkpoint1, null, seqNoMeasurement1.getAttributes()); - - TestMeasurement seqNoMeasurement2 = subs2.getMeasurements().get(0); - assertEquals(42L, seqNoMeasurement2.getValue()); - assertAttributes(checkpoint2, null, seqNoMeasurement2.getAttributes()); - - TestHistogram duration = meter.getHistograms().get("messaging.eventhubs.checkpoint.duration"); - assertEquals(2, duration.getMeasurements().size()); - - assertEquals(0d, duration.getMeasurements().get(0).getValue(), 1000d); - assertAttributes(checkpoint1, null, duration.getMeasurements().get(0).getAttributes()); - assertEquals(0d, duration.getMeasurements().get(1).getValue(), 1000d); - assertAttributes(checkpoint2, null, duration.getMeasurements().get(1).getAttributes()); - } - - - private void assertAttributes(Checkpoint checkpoint, String expectedStatus, Map attributes) { - if (expectedStatus == null) { - assertEquals(4, attributes.size()); - } else { - assertEquals(5, attributes.size()); - } - - assertEquals(checkpoint.getFullyQualifiedNamespace(), attributes.get("hostName")); - assertEquals(checkpoint.getEventHubName(), attributes.get("entityName")); - assertEquals(checkpoint.getPartitionId(), attributes.get("partitionId")); - assertEquals(checkpoint.getConsumerGroup(), attributes.get("consumerGroup")); - assertEquals(expectedStatus, attributes.get("status")); - } -} diff --git a/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/test/java/com/azure/messaging/eventhubs/checkpointstore/blob/TestMeterProvider.java b/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/test/java/com/azure/messaging/eventhubs/checkpointstore/blob/TestMeterProvider.java deleted file mode 100644 index 0bf36956c5c77..0000000000000 --- a/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/test/java/com/azure/messaging/eventhubs/checkpointstore/blob/TestMeterProvider.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.messaging.eventhubs.checkpointstore.blob; - -import com.azure.core.util.MetricsOptions; -import com.azure.core.util.metrics.Meter; -import com.azure.core.util.metrics.MeterProvider; - -public class TestMeterProvider implements MeterProvider { - - private final MeterFactory meterFactory; - public TestMeterProvider(MeterFactory meterFactory) { - this.meterFactory = meterFactory; - } - - @Override - public Meter createMeter(String libraryName, String libraryVersion, MetricsOptions options) { - return meterFactory.createMeter(libraryName, libraryVersion, options); - } - - @FunctionalInterface - interface MeterFactory { - Meter createMeter(String libraryName, String libraryVersion, MetricsOptions options); - } -} diff --git a/sdk/eventhubs/azure-messaging-eventhubs-stress/Chart.yaml b/sdk/eventhubs/azure-messaging-eventhubs-stress/Chart.yaml index f2b9fb40d3edd..8208de497de7f 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs-stress/Chart.yaml +++ b/sdk/eventhubs/azure-messaging-eventhubs-stress/Chart.yaml @@ -9,5 +9,5 @@ annotations: dependencies: - name: stress-test-addons - version: 0.2.0 - repository: https://stresstestcharts.blob.core.windows.net/helm/ + version: ~0.3.0 + repository: "@stress-test-charts" diff --git a/sdk/eventhubs/azure-messaging-eventhubs-stress/workbooks/details.json b/sdk/eventhubs/azure-messaging-eventhubs-stress/workbooks/details.json index 3d4d7ed53a7e0..67ce05464c57f 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs-stress/workbooks/details.json +++ b/sdk/eventhubs/azure-messaging-eventhubs-stress/workbooks/details.json @@ -52,7 +52,7 @@ "allowCustom": true }, "value": { - "durationMs": 900000 + "durationMs": 86400000 } }, { @@ -86,7 +86,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "let runs = customMetrics \r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" \r\n| extend runId = tostring(split(cloud_RoleName, \"-\")[2])\r\n| summarize start=min(timestamp), end=max(timestamp) by runId;\r\nlet runSpans = dependencies\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| extend runId = tostring(split(cloud_RoleName, \"-\")[2])\r\n| where cloud_RoleName startswith \"java-eventhubs\" and name == \"before run\"\r\n| extend annotation = coalesce(customDimensions[\"annotation\"], \"\")\r\n| extend packageVersion = coalesce(customDimensions[\"packageVersion\"], \"\")\r\n| distinct annotation, packageVersion, runId, pod=tostring(customDimensions[\"hostname\"]);\r\nruns \r\n| join kind = innerunique runSpans on runId\r\n| order by start desc\r\n| project-away runId1\r\n", + "query": "let runs = customMetrics \r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" \r\n| extend runId = tostring(split(cloud_RoleName, \"-\")[2])\r\n| summarize start=min(timestamp), duration=(max(timestamp) - min(timestamp)) by runId;\r\nlet runSpans = dependencies\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| extend runId = tostring(split(cloud_RoleName, \"-\")[2])\r\n| where cloud_RoleName startswith \"java-eventhubs\" and name == \"before run\"\r\n| extend packageVersion = coalesce(customDimensions[\"packageVersion\"], \"\")\r\n| distinct runId, pod=tostring(customDimensions[\"hostname\"]), packageVersion;\r\nruns \r\n| join kind = innerunique runSpans on runId\r\n| order by start desc\r\n| project-away runId1\r\n", "size": 0, "title": "Runs in {timeRange:label}", "noDataMessageStyle": 5, @@ -117,7 +117,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "let runId = \"{runId}\";\r\nlet metrics = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName endswith runId;\r\nlet testSpans = dependencies\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName endswith runId;\r\nlet errors = metrics\r\n| where name == \"test.run.errors\"\r\n| extend errorType = tostring(customDimensions[\"error.type\"])\r\n| summarize error_by_type=toint(sum(valueSum)) by errorType\r\n| summarize test_errors=make_bag(bag_pack(errorType, error_by_type))\r\n| evaluate narrow();\r\nlet sentCounter = metrics | where name == \"messaging.eventhubs.events.sent\"\r\n| where customDimensions[\"error.type\"] == \"\"\r\n| extend evenHubName = tostring(customDimensions[\"messaging.destination.name\"])\r\n| summarize sent = sum(valueSum) by evenHubName\r\n| summarize sent_mesages=make_bag(bag_pack(evenHubName, toint(sent)))\r\n| evaluate narrow();\r\nlet receivedCounter = metrics | where name == \"messaging.eventhubs.consumer.lag\"\r\n| summarize received_messages = sum(valueCount)\r\n| evaluate narrow();\r\nlet checkpoints = metrics \r\n| where name == \"messaging.eventhubs.checkpoint.duration\" and customDimensions[\"otel.status_code\"] == \"ok\" \r\n| summarize checkpoints = sum(valueCount)\r\n| evaluate narrow(); \r\nlet actualDuration = testSpans \r\n| where name == \"after run\"\r\n| summarize duration = max(toreal(customDimensions[\"durationMs\"])/1000) by cloud_RoleInstance\r\n| summarize actual_test_duration_in_sec = make_bag(bag_pack(cloud_RoleInstance, duration))\r\n| evaluate narrow();\r\nlet parameters = testSpans \r\n| where name == \"before run\" and cloud_RoleInstance endswith \"receiver\"\r\n| project parameters_common=bag_pack(\"pod\", customDimensions[\"hostname\"], \"packageVersion\", customDimensions[\"packageVersion\"]), parameters_receiver = bag_remove_keys(customDimensions, dynamic(['eventHubName', 'duration', 'sample.in', 'consumerGroupName', 'annotation', 'hostname', 'packageVersion', 'jreVendor', 'jreVersion']))\r\n| evaluate narrow();\r\nlet sender_parameters = testSpans \r\n| where name == \"before run\" and cloud_RoleInstance endswith \"sender\"\r\n| project parameters_sender = bag_remove_keys(customDimensions, dynamic(['eventHubName', 'duration', 'sample.in', 'annotation', 'hostname', 'packageVersion', 'consumerGroupName', 'jreVendor', 'jreVersion']))\r\n| evaluate narrow();\r\nparameters \r\n| union sender_parameters, sentCounter, receivedCounter, checkpoints, actualDuration, errors\r\n| project Property = Column, Value\r\n", + "query": "let runId = \"{runId}\";\r\nlet metrics = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName endswith runId;\r\nlet testSpans = dependencies\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName endswith runId;\r\nlet errors = metrics\r\n| where name == \"test.run.errors\"\r\n| extend errorType = tostring(customDimensions[\"error.type\"])\r\n| summarize error_by_type=toint(sum(valueSum)) by errorType\r\n| summarize test_errors=make_bag(bag_pack(errorType, error_by_type))\r\n| evaluate narrow();\r\nlet sentCounter = metrics | where name == \"messaging.client.published.messages\"\r\n| where customDimensions[\"error.type\"] == \"\"\r\n| extend evenHubName = tostring(customDimensions[\"messaging.destination.name\"])\r\n| summarize sent = sum(valueSum) by evenHubName\r\n| summarize sent_mesages=make_bag(bag_pack(evenHubName, toint(sent)))\r\n| evaluate narrow();\r\nlet receivedCounter = metrics | where name == \"messaging.client.consumed.messages\"\r\n| summarize received_messages = sum(valueSum)\r\n| evaluate narrow();\r\nlet checkpoints = metrics \r\n| where name == \"messaging.client.operation.duration\" and customDimensions[\"error.type\"] == \"\" and customDimensions[\"messaging.operation.name\"] == \"checkpoint\"\r\n| summarize checkpoints = sum(valueCount)\r\n| evaluate narrow(); \r\nlet actualDuration = testSpans \r\n| where name == \"after run\"\r\n| summarize duration = max(toreal(customDimensions[\"durationMs\"])/1000) by cloud_RoleInstance\r\n| summarize actual_test_duration_in_sec = make_bag(bag_pack(cloud_RoleInstance, duration))\r\n| evaluate narrow();\r\nlet parameters = testSpans \r\n| where name == \"before run\" and cloud_RoleInstance endswith \"receiver\"\r\n| project parameters_common=bag_pack(\"pod\", customDimensions[\"hostname\"], \"packageVersion\", customDimensions[\"packageVersion\"]), parameters_receiver = bag_remove_keys(customDimensions, dynamic(['eventHubName', 'duration', 'sample.in', 'consumerGroupName', 'annotation', 'hostname', 'packageVersion', 'jreVendor', 'jreVersion']))\r\n| evaluate narrow();\r\nlet sender_parameters = testSpans \r\n| where name == \"before run\" and cloud_RoleInstance endswith \"sender\"\r\n| project parameters_sender = bag_remove_keys(customDimensions, dynamic(['eventHubName', 'duration', 'sample.in', 'annotation', 'hostname', 'packageVersion', 'consumerGroupName', 'jreVendor', 'jreVersion']))\r\n| evaluate narrow();\r\nparameters \r\n| union sender_parameters, sentCounter, receivedCounter, checkpoints, actualDuration, errors\r\n| project Property = Column, Value\r\n", "size": 0, "showAnalytics": true, "title": "Test results", @@ -133,10 +133,10 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "// rate\r\nlet runId = \"{runId}\";\r\nlet role_name=strcat(\"java-eventhubs-\", runId);\r\nlet send = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.eventhubs.events.sent\" and cloud_RoleName == role_name and cloud_RoleInstance==\"sender\"\r\n| where customDimensions[\"otel.status_code\"] == \"ok\" or customDimensions[\"otel.status_code\"] == \"\"\r\n| summarize message_send=sum(valueSum) by bin(timestamp, 1m);\r\nlet receive = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.eventhubs.consumer.lag\"\r\n| where cloud_RoleName == role_name and cloud_RoleInstance==\"receiver\"\r\n| summarize receive = sum(valueCount) by bin(timestamp, 1m);\r\nlet settle = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.eventhubs.checkpoint.duration\"\r\n| where cloud_RoleName == role_name and cloud_RoleInstance==\"receiver\"\r\n| where customDimensions[\"otel.status_code\"] == \"ok\" or customDimensions[\"otel.status_code\"] == \"\"\r\n| summarize settle = sum(valueCount) by bin(timestamp, 1m);\r\nsend \r\n| join kind = fullouter (receive) on timestamp\r\n| join kind = fullouter (settle) on timestamp\r\n| project-away timestamp1, timestamp2\r\n| render timechart", + "query": "// rate\r\nlet runId = \"{runId}\";\r\nlet role_name=strcat(\"java-eventhubs-\", runId);\r\nlet send = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.client.published.messages\" and cloud_RoleName == role_name and cloud_RoleInstance==\"sender\"\r\n| where customDimensions[\"error.type\"] == \"\"\r\n| summarize message_send=sum(valueSum) by bin(timestamp, 1m);\r\nlet process = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.client.consumed.messages\"\r\n| where cloud_RoleName == role_name and cloud_RoleInstance==\"receiver\"\r\n| where customDimensions[\"error.type\"] == \"\"\r\n| summarize process = sum(valueSum) by bin(timestamp, 1m);\r\nlet checkpoint = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.client.operation.duration\"\r\n| where cloud_RoleName == role_name and cloud_RoleInstance==\"receiver\"\r\n| where customDimensions[\"error.type\"] == \"\" and customDimensions[\"messaging.operation.name\"] == \"checkpoint\"\r\n| summarize checkpoint = sum(valueCount) by bin(timestamp, 1m);\r\nsend \r\n| join kind = fullouter (process) on timestamp\r\n| join kind = fullouter (checkpoint) on timestamp\r\n| project-away timestamp1, timestamp2\r\n| render timechart", "size": 0, "aggregation": 3, - "title": "Send/receive/checkpoint success rate (per minute)", + "title": "Send/process/checkpointsuccess rate (per minute)", "noDataMessageStyle": 5, "queryType": 0, "resourceType": "microsoft.insights/components" @@ -152,7 +152,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "// duration\r\nlet runId = \"{runId}\";\r\nlet role_name=strcat(\"java-eventhubs-\", runId);\r\nlet amqp_send = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.az.amqp.producer.send.duration\"\r\n| where cloud_RoleName == role_name and cloud_RoleInstance == \"sender\"\r\n| where customDimensions[\"amqp.delivery_state\"] == \"accepted\"\r\n| summarize amqp_send = avg(valueSum/valueCount) by bin(timestamp, 1m); // amqp_send_avg = avg(valueSum/valueCount), amqp_send_max = max(valueMax) \r\nlet settle = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.eventhubs.checkpoint.duration\"\r\n| where cloud_RoleName == role_name and cloud_RoleInstance == \"receiver\"\r\n| where customDimensions[\"otel.status_code\"] == \"ok\" or customDimensions[\"otel.status_code\"] == \"\"\r\n| summarize settle = avg(valueSum/valueCount) by bin(timestamp, 1m); // settle_avg = avg(valueSum/valueCount), settle_max = max(valueMax)\r\nlet management = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"messaging.az.amqp.management.request.duration\"\r\n| where cloud_RoleName == role_name\r\n| where customDimensions[\"amqp.status_code\"] == \"accepted\"\r\n| summarize management = avg(valueSum/valueCount) by bin(timestamp, 1m); // management_avg = avg(valueSum/valueCount), management_max = max(valueMax)\r\namqp_send\r\n| join kind = fullouter (settle) on timestamp\r\n| join kind = fullouter (management) on timestamp\r\n| project-away timestamp1, timestamp2\r\n| render linechart", + "query": "// duration\r\nlet runId = \"{runId}\";\r\nlet role_name=strcat(\"java-eventhubs-\", runId);\r\ncustomMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName == role_name\r\n| where customDimensions[\"error.type\"] == \"\"\r\n| where name == \"messaging.client.operation.duration\" or name == \"messaging.process.duration\"\r\n| summarize durationMs = 1000 * avg(valueSum/valueCount) by tostring(customDimensions[\"messaging.operation.name\"]), bin(timestamp, 1m) \r\n| render timechart\r\n", "size": 0, "aggregation": 3, "title": "Duration (ms), average per minute", @@ -170,9 +170,9 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "let runId = \"{runId}\";\r\nlet role_name=strcat(\"java-eventhubs-\", runId);\r\nlet metrics = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName == role_name;\r\nlet send = metrics\r\n| where name == \"messaging.az.amqp.producer.send.duration\"\r\n| extend status = tostring(customDimensions[\"amqp.delivery_state\"])\r\n| where status != \"\" and status != \"accepted\"\r\n| summarize send_errors = sum(valueCount) by status;\r\nlet settle = metrics\r\n| where name == \"messaging.eventhubs.checkpoint.duration\"\r\n| extend status = tostring(customDimensions[\"otel.status_code\"])\r\n| where status != \"ok\" and status != \"\"\r\n| summarize settle_errors = sum(valueCount) by status;\r\nlet management = metrics\r\n| where name == \"messaging.az.amqp.management.request.duration\"\r\n| extend status = tostring(customDimensions[\"amqp.status_code\"])\r\n| where status != \"ok\" and status != \"accepted\"\r\n| summarize management_errors = sum(valueCount) by status;\r\nlet connection = metrics\r\n| where name == \"messaging.az.amqp.client.connections.closed\"\r\n| extend status = tostring(customDimensions[\"amqp.error_condition\"])\r\n| where status != \"ok\"\r\n| summarize connection_errors = sum(valueSum) by status;\r\nlet transport = metrics \r\n| where name == \"messaging.az.amqp.client.transport.errors\"\r\n| extend status = tostring(customDimensions[\"amqp.error_condition\"])\r\n| where status != \"ok\"\r\n| summarize transport_errors = sum(valueSum) by status;\r\nsend \r\n| join kind = fullouter (settle) on status\r\n| join kind = fullouter (management) on status\r\n| join kind = fullouter (connection) on status\r\n| join kind = fullouter (transport) on status\r\n| project status = coalesce(status, status1, status2, status3, status4), send_errors, settle_errors, management_errors, connection_errors, transport_errors\r\n", + "query": "let runId = \"{runId}\";\r\nlet role_name=strcat(\"java-eventhubs-\", runId);\r\ncustomMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName == role_name\r\n| where name == \"messaging.client.operation.duration\" or name == \"messaging.process.duration\"\r\n| extend errorType = tostring(customDimensions[\"error.type\"])\r\n| where errorType != \"\"\r\n| summarize errors = sum(valueCount) by errorType, bin(timestamp, 1m)\r\n| render timechart", "size": 0, - "title": "Send/receive/checkpoint error rate (per minute)", + "title": "Send/process/checkpoint error rate (per minute)", "queryType": 0, "resourceType": "microsoft.insights/components" }, @@ -187,7 +187,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "let runId = \"{runId}\";\r\ncustomMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"process.runtime.jvm.memory.usage\" and customDimensions[\"type\"]==\"heap\"\r\n| summarize heap_memory_used=sum(valueSum/valueCount) by bin(timestamp, 1m), cloud_RoleInstance\r\n| render timechart\r\n", + "query": "let runId = \"{runId}\";\r\ncustomMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"jvm.memory.used\" and customDimensions[\"jvm.memory.type\"]==\"heap\"\r\n| summarize heap_memory_used=sum(valueSum/valueCount) by bin(timestamp, 1m), cloud_RoleInstance\r\n| render timechart\r\n", "size": 0, "aggregation": 3, "title": "Heap memory used (MB)", @@ -204,7 +204,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "let runId = \"{runId}\";\r\ncustomMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"process.runtime.jvm.cpu.utilization\"\r\n| summarize cpu_time_percent=avg(value) * 100 by bin(timestamp, 1m), cloud_RoleInstance\r\n| render timechart\r\n", + "query": "let runId = \"{runId}\";\r\ncustomMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"jvm.cpu.recent_utilization\"\r\n| summarize cpu_time_percent=avg(value) * 100 by bin(timestamp, 1m), cloud_RoleInstance\r\n| render timechart\r\n", "size": 0, "aggregation": 3, "title": "CPU %", @@ -221,7 +221,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"process.runtime.jvm.threads.count\"\r\n| summarize max_thread_count=max(valueMax) by bin(timestamp, 1m), cloud_RoleInstance\r\n| render timechart\r\n", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"jvm.thread.count\"\r\n| summarize max_thread_count=max(valueMax) by bin(timestamp, 1m), cloud_RoleInstance\r\n| render timechart\r\n", "size": 0, "aggregation": 2, "title": "Thread count", @@ -238,7 +238,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"process.runtime.jvm.gc.duration\" \r\n| extend gc_type=tostring(customDimensions[\"gc\"])\r\n| summarize gc_percentage=sum(valueSum) / 60 * 100 by cloud_RoleInstance, bin(timestamp, 1m)\r\n| render timechart\r\n", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-eventhubs\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"jvm.gc.duration\" \r\n| extend type = strcat(cloud_RoleInstance, \"_\", tostring(customDimensions[\"jvm.gc.name\"]))\r\n| summarize gc_percentage=sum(valueSum) / 60 * 100 by type, bin(timestamp, 1m)\r\n| render timechart\r\n", "size": 0, "aggregation": 3, "title": "% of time spent in GC", @@ -255,7 +255,7 @@ "type": 3, "content": { "version": "KqlItem/1.0", - "query": "let runId = \"{runId}\";\r\nlet metrics = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName endswith runId;\r\nlet send = metrics\r\n| where name == \"messaging.eventhubs.events.sent\"\r\n| extend status = tostring(customDimensions[\"otel.status_code\"])\r\n| where status != \"\" and status != \"ok\"\r\n| summarize send_errors = sum(valueSum) by bin(timestamp, 1m);\r\nlet settle = metrics\r\n| where name == \"messaging.eventhubs.checkpoint.duration\"\r\n| extend status = tostring(customDimensions[\"otel.status_code\"])\r\n| where status != \"ok\" and status != \"\"\r\n| summarize settle_errors = sum(valueCount) by bin(timestamp, 1m);\r\nlet management = metrics\r\n| where name == \"messaging.az.amqp.management.request.duration\"\r\n| extend status = tostring(customDimensions[\"amqp.status_code\"])\r\n| where status != \"ok\" and status != \"accepted\"\r\n| summarize management_errors = sum(valueCount) by bin(timestamp, 1m);\r\nlet connection = metrics\r\n| where name == \"messaging.az.amqp.client.connections.closed\"\r\n| extend status = tostring(customDimensions[\"amqp.error_condition\"])\r\n| where status != \"ok\"\r\n| summarize connection_errors = sum(valueSum) by bin(timestamp, 1m);\r\nlet transport = metrics \r\n| where name == \"messaging.az.amqp.client.transport.errors\"\r\n| extend status = tostring(customDimensions[\"amqp.error_condition\"])\r\n| where status != \"ok\"\r\n| summarize transport_errors = sum(valueSum) by bin(timestamp, 1m);\r\nlet testErrors = metrics\r\n| where name == \"test.run.errors\"\r\n| extend errorType = tostring(customDimensions[\"error.type\"])\r\n| summarize test_errors=sum(valueSum) by errorType, bin(timestamp, 1m);\r\nlet logs = traces \r\n| union exceptions\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName endswith \"{runId}\" and severityLevel > 1\r\n| summarize log_errors = count() by bin(timestamp, 1m);\r\nsend \r\n| join kind = fullouter (testErrors) on timestamp\r\n| join kind = fullouter (settle) on timestamp\r\n| join kind = fullouter (management) on timestamp\r\n| join kind = fullouter (connection) on timestamp\r\n| join kind = fullouter (transport) on timestamp\r\n| join kind = fullouter (logs) on timestamp\r\n| project timestamp = coalesce(timestamp, timestamp1, timestamp2, timestamp3, timestamp4, timestamp5, timestamp6), test_errors, send_errors, settle_errors, management_errors, connection_errors, transport_errors, log_errors\r\n| render timechart\r\n", + "query": "let runId = \"{runId}\";\r\nlet metrics = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName endswith runId;\r\nlet send = metrics\r\n| where name == \"messaging.eventhubs.events.sent\"\r\n| extend status = tostring(customDimensions[\"otel.status_code\"])\r\n| where status != \"\" and status != \"ok\"\r\n| summarize send_errors = sum(valueSum) by bin(timestamp, 1m);\r\nlet settle = metrics\r\n| where name == \"messaging.settle.duration\"\r\n| extend status = tostring(customDimensions[\"error.type\"])\r\n| where status != \"\"\r\n| summarize settle_errors = sum(valueCount) by bin(timestamp, 1m);\r\nlet management = metrics\r\n| where name == \"messaging.az.amqp.management.request.duration\"\r\n| extend status = tostring(customDimensions[\"amqp.status_code\"])\r\n| where status != \"ok\" and status != \"accepted\" and status != \"\"\r\n| summarize management_errors = sum(valueCount) by bin(timestamp, 1m);\r\nlet connection = metrics\r\n| where name == \"connection.client.connection_duration\"\r\n| extend errorType = tostring(customDimensions[\"error.type\"])\r\n| where errorType != \"\"\r\n| summarize connection_errors = sum(valueCount) by bin(timestamp, 1m);\r\nlet session = metrics \r\n| where name == \"amqp.client.session.duration\"\r\n| extend errorType = tostring(customDimensions[\"error.type\"])\r\n| where errorType != \"\"\r\n| summarize session_errors = sum(valueCount) by bin(timestamp, 1m);\r\nlet link = metrics \r\n| where name == \"amqp.client.link.duration\"\r\n| extend errorType = tostring(customDimensions[\"error.type\"])\r\n| where errorType != \"\"\r\n| summarize link_errors = sum(valueCount) by bin(timestamp, 1m);\r\nlet testErrors = metrics\r\n| where name == \"test.run.errors\"\r\n| where tostring(customDimensions[\"error.type\"]) != \"\"\r\n| summarize test_errors=sum(valueSum) by bin(timestamp, 1m);\r\nsend \r\n| join kind = fullouter (testErrors) on timestamp\r\n| join kind = fullouter (settle) on timestamp\r\n| join kind = fullouter (management) on timestamp\r\n| join kind = fullouter (connection) on timestamp\r\n| join kind = fullouter (link) on timestamp\r\n| join kind = fullouter (session) on timestamp\r\n| project timestamp = coalesce(timestamp, timestamp1, timestamp2, timestamp3, timestamp4, timestamp5), test_errors, send_errors, settle_errors, management_errors, connection_errors, link_errors, session_errors\r\n| render timechart\r\n", "size": 0, "title": "Errors by time", "queryType": 0, @@ -276,7 +276,20 @@ "title": "Warnings and errors in logs (sampled, 1%)", "queryType": 0, "resourceType": "microsoft.insights/components", - "sortBy": [] + "gridSettings": { + "sortBy": [ + { + "itemKey": "occurences", + "sortOrder": 2 + } + ] + }, + "sortBy": [ + { + "itemKey": "occurences", + "sortOrder": 2 + } + ] }, "customWidth": "60", "name": "query - 6", diff --git a/sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md b/sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md index 90904c781501a..6161f64c1851f 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md +++ b/sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md @@ -7,9 +7,17 @@ - Integrated RequestResponseChannelCache (CBS, Management channel cache) and ReactorSessionCache, these caches are activated when V2 stack is opted-in using the configuration `com.azure.messaging.eventhubs.v2`. ([39107](https://github.com/Azure/azure-sdk-for-java/pull/39107)), ([41805](https://github.com/Azure/azure-sdk-for-java/pull/41805)) - Add `EventProcessorClient.stop(Duration timeout)` to stop the `EventProcessorClient` and await it to shut down. The `EventProcessorClient.stop()` method now will wait for up to the default timeout (10 seconds) waiting for the processor to stop. ([#41878](https://github.com/Azure/azure-sdk-for-java/pull/41878)) +- Observability improvements + - Added span for update checkpoint call. + - Added partitionId and consumer group (when available) to span and metric attributes. ### Breaking Changes +- Updated distributed traces and metrics to follow OpenTelemetry semantic conventions 1.27.0. + Please refer to [OpenTelemetry messaging semantic conventions](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/messaging) for more details. + - Span names now follow `{entity name} {operaiton name}` pattern + - Updated metric names and attributes + ### Bugs Fixed - Fixes the event size computation in EventHubSerializer to include size of delivery annotations. ([41605](https://github.com/Azure/azure-sdk-for-java/issues/41605)) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubBufferedPartitionProducer.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubBufferedPartitionProducer.java index 0227ff41c78bb..ccb9e64fc6240 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubBufferedPartitionProducer.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubBufferedPartitionProducer.java @@ -82,7 +82,7 @@ class EventHubBufferedPartitionProducer implements Closeable { .publishOn(Schedulers.boundedElastic(), 1) .subscribeWith(publishResultSubscriber); - this.tracer = new EventHubsTracer(tracer, client.getFullyQualifiedNamespace(), client.getEventHubName()); + this.tracer = new EventHubsTracer(tracer, client.getFullyQualifiedNamespace(), client.getEventHubName(), null); } /** diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubClientBuilder.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubClientBuilder.java index e0a18c90d89df..a8a05060a743f 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubClientBuilder.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubClientBuilder.java @@ -986,9 +986,7 @@ EventHubAsyncClient buildAsyncClient() { prefetchCount = DEFAULT_PREFETCH_COUNT; } - final Meter meter = MeterProvider.getDefaultProvider().createMeter(LIBRARY_NAME, LIBRARY_VERSION, - clientOptions == null ? null : clientOptions.getMetricsOptions()); - + final Meter meter = createMeter(); final MessageSerializer messageSerializer = new EventHubMessageSerializer(); final ConnectionCacheWrapper processor; @@ -1091,6 +1089,11 @@ Tracer createTracer() { AZ_NAMESPACE_VALUE, clientOptions == null ? null : clientOptions.getTracingOptions()); } + Meter createMeter() { + return MeterProvider.getDefaultProvider().createMeter(LIBRARY_NAME, LIBRARY_VERSION, + clientOptions == null ? null : clientOptions.getMetricsOptions()); + } + private EventHubConnectionProcessor buildConnectionProcessor(MessageSerializer messageSerializer, Meter meter) { final ConnectionOptions connectionOptions = getConnectionOptions(); final Supplier getEventHubName = () -> { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClient.java index 14c78f21f4d06..f45b50cc06b0c 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClient.java @@ -40,6 +40,8 @@ import static com.azure.messaging.eventhubs.implementation.ClientConstants.LINK_NAME_KEY; import static com.azure.messaging.eventhubs.implementation.ClientConstants.PARTITION_ID_KEY; import static com.azure.messaging.eventhubs.implementation.ClientConstants.SIGNAL_TYPE_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_EVENT_HUB_PROPERTIES; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_PARTITION_PROPERTIES; /** * An asynchronous consumer responsible for reading {@link EventData} from either a specific Event Hub partition @@ -308,7 +310,7 @@ boolean isV2() { public Mono getEventHubProperties() { return instrumentation.getTracer().traceMono(connectionProcessor.getManagementNodeWithRetries() .flatMap(EventHubManagementNode::getEventHubProperties), - "EventHubs.getEventHubProperties"); + GET_EVENT_HUB_PROPERTIES, null); } /** @@ -341,7 +343,7 @@ public Mono getPartitionProperties(String partitionId) { return instrumentation.getTracer().traceMono( connectionProcessor.getManagementNodeWithRetries().flatMap(node -> node.getPartitionProperties(partitionId)), - "EventHubs.getPartitionProperties"); + GET_PARTITION_PROPERTIES, partitionId); } /** diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java index d000b0a812ab4..e36e14320674a 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java @@ -6,12 +6,11 @@ import com.azure.core.annotation.ReturnType; import com.azure.core.annotation.ServiceClient; import com.azure.core.annotation.ServiceMethod; -import com.azure.core.util.Context; import com.azure.core.util.IterableStream; import com.azure.core.util.logging.ClientLogger; import com.azure.messaging.eventhubs.implementation.SynchronousEventSubscriber; import com.azure.messaging.eventhubs.implementation.SynchronousReceiveWork; -import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer; +import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; import com.azure.messaging.eventhubs.models.EventPosition; import com.azure.messaging.eventhubs.models.PartitionEvent; import com.azure.messaging.eventhubs.models.ReceiveOptions; @@ -20,7 +19,6 @@ import java.io.Closeable; import java.time.Duration; -import java.time.Instant; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; @@ -126,7 +124,7 @@ public class EventHubConsumerClient implements Closeable { private final ReceiveOptions defaultReceiveOptions = new ReceiveOptions(); private final Duration timeout; private final AtomicInteger idGenerator = new AtomicInteger(); - private final EventHubsTracer tracer; + private final EventHubsConsumerInstrumentation instrumentation; private final SynchronousPartitionReceiver syncReceiver; EventHubConsumerClient(EventHubConsumerAsyncClient consumer, Duration tryTimeout) { @@ -134,7 +132,7 @@ public class EventHubConsumerClient implements Closeable { this.consumer = Objects.requireNonNull(consumer, "'consumer' cannot be null."); this.timeout = tryTimeout; - this.tracer = consumer.getInstrumentation().getTracer(); + this.instrumentation = consumer.getInstrumentation(); this.syncReceiver = new SynchronousPartitionReceiver(consumer); // used in V2 mode. } @@ -262,19 +260,16 @@ public IterableStream receiveFromPartition(String partitionId, i } if (consumer.isV2()) { + // TODO(limolkova) instrument return syncReceiver.receive(partitionId, startingPosition, defaultReceiveOptions, maximumMessageCount, maximumWaitTime); } - Instant startTime = tracer.isEnabled() ? Instant.now() : null; + Flux events = + Flux.create(emitter -> queueWork(partitionId, maximumMessageCount, startingPosition, maximumWaitTime, defaultReceiveOptions, + emitter)); - Flux events = Flux.create(emitter -> { - queueWork(partitionId, maximumMessageCount, startingPosition, maximumWaitTime, defaultReceiveOptions, - emitter); - }); - - events = tracer.reportSyncReceiveSpan("EventHubs.receiveFromPartition", startTime, events, Context.NONE); - return new IterableStream<>(events); + return new IterableStream<>(instrumentation.syncReceive(events, partitionId)); } /** @@ -320,16 +315,16 @@ public IterableStream receiveFromPartition(String partitionId, i } if (consumer.isV2()) { + // TODO (instrument) return syncReceiver.receive(partitionId, startingPosition, receiveOptions, maximumMessageCount, maximumWaitTime); } - Instant startTime = tracer.isEnabled() ? Instant.now() : null; Flux events = Flux.create(emitter -> { queueWork(partitionId, maximumMessageCount, startingPosition, maximumWaitTime, receiveOptions, emitter); }); - events = tracer.reportSyncReceiveSpan("EventHubs.receiveFromPartition", startTime, events, Context.NONE); - return new IterableStream<>(events); + + return new IterableStream<>(instrumentation.syncReceive(events, partitionId)); } /** diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClient.java index c3028e6b63664..79c65e1db2617 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClient.java @@ -44,6 +44,8 @@ import static com.azure.messaging.eventhubs.implementation.ClientConstants.MAX_MESSAGE_LENGTH_BYTES; import static com.azure.messaging.eventhubs.implementation.ClientConstants.PARTITION_ID_KEY; import static com.azure.messaging.eventhubs.implementation.ClientConstants.PARTITION_KEY_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_EVENT_HUB_PROPERTIES; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_PARTITION_PROPERTIES; /** *

An asynchronous producer responsible for transmitting {@link EventData} to a specific Event Hub, grouped @@ -299,7 +301,7 @@ public String getEventHubName() { public Mono getEventHubProperties() { return instrumentation.getTracer().traceMono( connectionProcessor.getManagementNodeWithRetries().flatMap(EventHubManagementNode::getEventHubProperties), - "EventHubs.getEventHubProperties"); + GET_EVENT_HUB_PROPERTIES, null); } /** @@ -324,7 +326,7 @@ public Flux getPartitionIds() { public Mono getPartitionProperties(String partitionId) { return instrumentation.getTracer().traceMono( connectionProcessor.getManagementNodeWithRetries().flatMap(node -> node.getPartitionProperties(partitionId)), - "EventHubs.getPartitionProperties"); + GET_PARTITION_PROPERTIES, partitionId); } /** @@ -614,7 +616,7 @@ public Mono send(EventDataBatch batch) { .publishOn(scheduler); // important to end spans after metrics are reported so metrics get relevant context for exemplars. - return instrumentation.onSendBatch(send, batch, "EventHubs.send"); + return instrumentation.sendBatch(send, batch); } private Mono sendInternal(Flux events, SendOptions options) { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java index d101e8270cb14..18588abff5978 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java @@ -10,58 +10,66 @@ import com.azure.core.util.tracing.Tracer; import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsMetricsProvider; import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer; +import com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationScope; +import com.azure.messaging.eventhubs.implementation.instrumentation.OperationName; import reactor.core.publisher.Mono; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.MESSAGING_BATCH_SIZE_ATTRIBUTE_NAME; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.REACTOR_PARENT_TRACE_CONTEXT_KEY; -class EventHubsProducerInstrumentation { +import java.util.function.BiConsumer; + +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_BATCH_MESSAGE_COUNT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_PARTITION_ID; +class EventHubsProducerInstrumentation { private final EventHubsTracer tracer; private final EventHubsMetricsProvider meter; EventHubsProducerInstrumentation(Tracer tracer, Meter meter, String fullyQualifiedName, String entityName) { - this.tracer = new EventHubsTracer(tracer, fullyQualifiedName, entityName); + this.tracer = new EventHubsTracer(tracer, fullyQualifiedName, entityName, null); this.meter = new EventHubsMetricsProvider(meter, fullyQualifiedName, entityName, null); } - Mono onSendBatch(Mono publisher, EventDataBatch batch, String spanName) { - if (!tracer.isEnabled() && !meter.isSendCountEnabled()) { + Mono sendBatch(Mono publisher, EventDataBatch batch) { + if (!isEnabled()) { return publisher; } - if (tracer.isEnabled()) { - return publisher - .doOnEach(signal -> { - if (signal.isOnComplete() || signal.isOnError()) { - Context span = signal.getContextView().getOrDefault(REACTOR_PARENT_TRACE_CONTEXT_KEY, Context.NONE); - meter.reportBatchSend(batch.getCount(), batch.getPartitionId(), signal.getThrowable(), span); - tracer.endSpan(signal.getThrowable(), span, null); - } - }) - .contextWrite(reactor.util.context.Context.of(REACTOR_PARENT_TRACE_CONTEXT_KEY, startPublishSpanWithLinks(spanName, batch, Context.NONE))); - } else { - return publisher - .doOnEach(signal -> { - if (signal.isOnComplete() || signal.isOnError()) { - meter.reportBatchSend(batch.getCount(), batch.getPartitionId(), signal.getThrowable(), Context.NONE); - } - }); - } + BiConsumer reportMetricsCallback = (m, s) -> + m.reportBatchSend(batch.getCount(), batch.getPartitionId(), s); + + + return Mono.using( + () -> new InstrumentationScope(tracer, meter, reportMetricsCallback) + .setSpan(startPublishSpanWithLinks(batch, Context.NONE)), + scope -> publisher + .doOnError(scope::setError) + .doOnCancel(scope::setCancelled), + InstrumentationScope::close); } public EventHubsTracer getTracer() { return tracer; } - private Context startPublishSpanWithLinks(String spanName, EventDataBatch batch, Context context) { - StartSpanOptions startOptions = tracer.createStartOption(SpanKind.CLIENT, EventHubsTracer.OperationName.PUBLISH); + private Context startPublishSpanWithLinks(EventDataBatch batch, Context context) { + if (!tracer.isEnabled()) { + return context; + } + + StartSpanOptions startOptions = tracer.createStartOptions(SpanKind.CLIENT, OperationName.SEND, null); if (batch != null) { - startOptions.setAttribute(MESSAGING_BATCH_SIZE_ATTRIBUTE_NAME, batch.getCount()); + startOptions.setAttribute(MESSAGING_BATCH_MESSAGE_COUNT, batch.getCount()); + if (batch.getPartitionId() != null) { + startOptions.setAttribute(MESSAGING_DESTINATION_PARTITION_ID, batch.getPartitionId()); + } for (EventData event : batch.getEvents()) { - startOptions.addLink(tracer.createLink(event.getProperties(), null, event.getContext())); + startOptions.addLink(tracer.createProducerLink(event.getProperties(), event.getContext())); } } - return tracer.startSpan(spanName, startOptions, context); + return tracer.startSpan(OperationName.SEND, startOptions, context); + } + + private boolean isEnabled() { + return tracer.isEnabled() || meter.isEnabled(); } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java index 58119671bf691..2a842c89231ab 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java @@ -5,9 +5,11 @@ import com.azure.core.annotation.ServiceClient; import com.azure.core.util.logging.ClientLogger; +import com.azure.core.util.metrics.Meter; import com.azure.core.util.tracing.Tracer; import com.azure.messaging.eventhubs.implementation.PartitionProcessor; -import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer; +import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; +import com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentedCheckpointStore; import com.azure.messaging.eventhubs.models.ErrorContext; import com.azure.messaging.eventhubs.models.PartitionOwnership; import reactor.core.publisher.Flux; @@ -108,7 +110,7 @@ public class EventProcessorClient { */ EventProcessorClient(EventHubClientBuilder eventHubClientBuilder, Supplier partitionProcessorFactory, CheckpointStore checkpointStore, - Consumer processError, Tracer tracer, EventProcessorClientOptions processorClientOptions) { + Consumer processError, Tracer tracer, Meter meter, EventProcessorClientOptions processorClientOptions) { Objects.requireNonNull(eventHubClientBuilder, "eventHubClientBuilder cannot be null."); this.processorClientOptions = Objects.requireNonNull(processorClientOptions, @@ -119,7 +121,6 @@ public class EventProcessorClient { final EventHubAsyncClient eventHubAsyncClient = eventHubClientBuilder.buildAsyncClient(); - this.checkpointStore = Objects.requireNonNull(checkpointStore, "checkpointStore cannot be null"); this.identifier = eventHubAsyncClient.getIdentifier(); Map loggingContext = new HashMap<>(); @@ -131,9 +132,14 @@ public class EventProcessorClient { this.consumerGroup = processorClientOptions.getConsumerGroup().toLowerCase(Locale.ROOT); this.loadBalancerUpdateInterval = processorClientOptions.getLoadBalancerUpdateInterval(); - final EventHubsTracer eventHubsTracer = new EventHubsTracer(tracer, fullyQualifiedNamespace, eventHubName); - this.partitionPumpManager = new PartitionPumpManager(checkpointStore, partitionProcessorFactory, - eventHubClientBuilder, eventHubsTracer, processorClientOptions); + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation( + tracer, meter, fullyQualifiedNamespace, eventHubName, consumerGroup, true); + + Objects.requireNonNull(checkpointStore, "checkpointStore cannot be null"); + this.checkpointStore = InstrumentedCheckpointStore.create(checkpointStore, instrumentation); + + this.partitionPumpManager = new PartitionPumpManager(this.checkpointStore, partitionProcessorFactory, + eventHubClientBuilder, instrumentation, processorClientOptions); this.partitionBasedLoadBalancer = new PartitionBasedLoadBalancer(this.checkpointStore, eventHubAsyncClient, diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClientBuilder.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClientBuilder.java index 9d13d35036579..e89a8479ead72 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClientBuilder.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClientBuilder.java @@ -877,7 +877,7 @@ public EventProcessorClient buildEventProcessorClient() { final EventHubClientBuilder builder = copyOptions(eventHubClientBuilder); return new EventProcessorClient(builder, getPartitionProcessorSupplier(), checkpointStore, - processError, eventHubClientBuilder.createTracer(), processorOptions); + processError, eventHubClientBuilder.createTracer(), eventHubClientBuilder.createMeter(), processorOptions); } private Supplier getPartitionProcessorSupplier() { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/PartitionPumpManager.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/PartitionPumpManager.java index 438701fe52c6e..de1858401fa8c 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/PartitionPumpManager.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/PartitionPumpManager.java @@ -3,13 +3,13 @@ package com.azure.messaging.eventhubs; -import com.azure.core.util.Context; import com.azure.core.util.logging.ClientLogger; import com.azure.core.util.logging.LogLevel; import com.azure.messaging.eventhubs.implementation.PartitionProcessor; import com.azure.messaging.eventhubs.implementation.PartitionProcessorException; import com.azure.messaging.eventhubs.implementation.ReactorShim; -import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer; +import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; +import com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationScope; import com.azure.messaging.eventhubs.models.Checkpoint; import com.azure.messaging.eventhubs.models.CloseContext; import com.azure.messaging.eventhubs.models.CloseReason; @@ -64,7 +64,7 @@ class PartitionPumpManager { private final Supplier partitionProcessorFactory; private final EventHubClientBuilder eventHubClientBuilder; private final int prefetch; - private final EventHubsTracer tracer; + private final EventHubsConsumerInstrumentation instrumentation; private final EventProcessorClientOptions options; /** @@ -75,11 +75,11 @@ class PartitionPumpManager { * PartitionProcessor} when new partition pumps are started. * @param eventHubClientBuilder The client builder used to create new clients (and new connections) for each * partition processed by this {@link EventProcessorClient}. - * @param tracer Tracing helper. + * @param instrumentation Tracing and metrics helper. * @param options Configuration options. */ PartitionPumpManager(CheckpointStore checkpointStore, Supplier partitionProcessorFactory, - EventHubClientBuilder eventHubClientBuilder, EventHubsTracer tracer, EventProcessorClientOptions options) { + EventHubClientBuilder eventHubClientBuilder, EventHubsConsumerInstrumentation instrumentation, EventProcessorClientOptions options) { this.checkpointStore = checkpointStore; this.partitionProcessorFactory = partitionProcessorFactory; this.eventHubClientBuilder = eventHubClientBuilder; @@ -87,7 +87,7 @@ class PartitionPumpManager { this.prefetch = eventHubClientBuilder.getPrefetchCount() == null ? EventHubClientBuilder.DEFAULT_PREFETCH_COUNT : eventHubClientBuilder.getPrefetchCount(); - this.tracer = tracer; + this.instrumentation = instrumentation; } /** @@ -279,9 +279,7 @@ private void processEvent(PartitionContext partitionContext, PartitionProcessor private void processEvents(PartitionContext partitionContext, PartitionProcessor partitionProcessor, PartitionPump partitionPump, List partitionEventBatch) { - Throwable exception = null; - Context span = null; - AutoCloseable scope = null; + InstrumentationScope scope = null; try { if (options.isBatchReceiveMode()) { @@ -300,8 +298,7 @@ private void processEvents(PartitionContext partitionContext, PartitionProcessor EventBatchContext eventBatchContext = new EventBatchContext(partitionContext, eventDataList, checkpointStore, enqueuedEventProperties); - span = tracer.startProcessSpan("EventHubs.process", eventDataList, Context.NONE); - scope = tracer.makeSpanCurrent(span); + scope = instrumentation.startProcess(eventBatchContext); if (LOGGER.canLogAtLevel(LogLevel.VERBOSE)) { LOGGER.atVerbose() .addKeyValue(PARTITION_ID_KEY, partitionContext.getPartitionId()) @@ -329,18 +326,22 @@ private void processEvents(PartitionContext partitionContext, PartitionProcessor EventContext eventContext = new EventContext(partitionContext, eventData, checkpointStore, enqueuedEventProperties); - span = tracer.startProcessSpan("EventHubs.process", eventData, Context.NONE); - scope = tracer.makeSpanCurrent(span); + + scope = instrumentation.startProcess(eventContext); processEvent(partitionContext, partitionProcessor, eventContext); } } catch (Throwable throwable) { - exception = throwable; + if (scope != null) { + scope.setError(throwable); + } /* user code for event processing threw an exception - log and bubble up */ throw LOGGER.logExceptionAsError(new PartitionProcessorException("Error in event processing callback", throwable)); } finally { - tracer.endSpan(exception, span, scope); + if (scope != null) { + scope.close(); + } } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java new file mode 100644 index 0000000000000..745663c20fde0 --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs; + +import com.azure.core.amqp.implementation.WindowedSubscriber; +import com.azure.core.amqp.implementation.WindowedSubscriber.WindowedSubscriberOptions; +import com.azure.core.util.IterableStream; +import com.azure.core.util.logging.ClientLogger; +import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; +import com.azure.messaging.eventhubs.models.EventPosition; +import com.azure.messaging.eventhubs.models.PartitionEvent; +import com.azure.messaging.eventhubs.models.ReceiveOptions; +import reactor.core.publisher.Flux; + +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; + +import static com.azure.messaging.eventhubs.implementation.ClientConstants.PARTITION_ID_KEY; + +/** + * A type that channels synchronous receive requests to a backing asynchronous receiver client. + */ +final class SynchronousReceiver { + private static final String TERMINAL_MESSAGE; + private static final WindowedSubscriber DISPOSED; + static { + TERMINAL_MESSAGE = "The receiver client is terminated. Re-create the client to continue receive attempt."; + DISPOSED = Flux.error(new RuntimeException("Disposed.")) + .subscribeWith(new WindowedSubscriber<>(new HashMap<>(0), TERMINAL_MESSAGE, new WindowedSubscriberOptions<>())); + } + private static final String SYNC_RECEIVE_SPAN_NAME = "EventHubs.receiveFromPartition"; + private final ClientLogger logger; + private final EventHubConsumerAsyncClient asyncClient; + private final EventHubsConsumerInstrumentation instrumentation; + private final AtomicReference> subscriber = new AtomicReference<>(null); + + /** + * Creates a SynchronousReceiver. + * + * @param logger the logger to use. + * @param asyncClient the backing asynchronous client to connect to the broker, delegate message requesting and receive. + */ + SynchronousReceiver(ClientLogger logger, EventHubConsumerAsyncClient asyncClient) { + this.logger = Objects.requireNonNull(logger, "'logger' cannot be null."); + this.asyncClient = Objects.requireNonNull(asyncClient, "'asyncClient' cannot be null."); + this.instrumentation = asyncClient.getInstrumentation(); + } + + /** + * Request a specified number of event and obtain an {@link IterableStream} streaming the received event. + * + * @param partitionId Identifier of the partition to read events from. + * @param startingPosition Position within the Event Hub partition to begin consuming events. + * @param receiveOptions Options when receiving events from the partition. + * @param maxEvents the maximum number of event to receive. + * @param maxWaitTime the upper bound for the time to wait to receive the requested number of event. + * + * @return an {@link IterableStream} of at most {@code maxEvents} event. + */ + IterableStream receive(String partitionId, EventPosition startingPosition, + ReceiveOptions receiveOptions, int maxEvents, Duration maxWaitTime) { + Objects.requireNonNull(partitionId, "'partitionId' cannot be null."); + Objects.requireNonNull(startingPosition, "'startingPosition' cannot be null."); + Objects.requireNonNull(receiveOptions, "'receiveOptions' cannot be null."); + + final WindowedSubscriber s = subscriber.get(); + if (s != null) { + return s.enqueueRequest(maxEvents, maxWaitTime); + } else { + return subscribeOnce(partitionId, startingPosition, receiveOptions).enqueueRequest(maxEvents, maxWaitTime); + } + } + + /** + * Disposes the SynchronousReceiver. + *

+ * Once disposed, the {@link IterableStream} for any future or pending receive requests will receive terminated error. + *

+ */ + void dispose() { + final WindowedSubscriber s = subscriber.getAndSet(DISPOSED); + if (s != null) { + s.dispose(); + } + } + + /** + * Obtain a {@link WindowedSubscriber} that is subscribed to the asynchronous message stream produced by the backing + * asynchronous client. + *

+ * The method subscribes only once and cache the subscriber. + *

+ *

+ * The subscriber exposes synchronous receive API and channels receive requests to the backing asynchronous message + * stream. + *

+ * + * @param partitionId Identifier of the partition to read events from. + * @param startingPosition Position within the Event Hub partition to begin consuming events. + * @param receiveOptions Options when receiving events from the partition. + * + * @return the subscriber to channel synchronous receive requests to the upstream. + */ + private WindowedSubscriber subscribeOnce(String partitionId, EventPosition startingPosition, + ReceiveOptions receiveOptions) { + if (!asyncClient.isV2()) { + throw logger.logExceptionAsError(new UnsupportedOperationException("SynchronousReceiver requires v2 mode.")); + } + final WindowedSubscriber s = createSubscriber(partitionId); + if (subscriber.compareAndSet(null, s)) { + // In case of concurrent invocation, the 's' created by the thread which lost the CAS race is eligible for GC. + // There is no leak, as 's' is in mere constructed state and don’t own any leak-able resource yet. + final Flux upstream = asyncClient.receiveFromPartition(partitionId, startingPosition, receiveOptions); + upstream.subscribeWith(s); + } + return subscriber.get(); + } + + /** + * Create a {@link WindowedSubscriber} capable of bridging synchronous receive requests to an upstream of + * asynchronous event. + * + * @param partitionId Identifier of the partition to read events from. + * + * @return The subscriber. + */ + private WindowedSubscriber createSubscriber(String partitionId) { + final WindowedSubscriberOptions options = new WindowedSubscriberOptions<>(); + options.setWindowDecorator(toDecorate -> { + // Decorates the provided 'toDecorate' flux for tracing the signals (events, termination) it produces. + return toDecorate; + //return instrumentation.reportSyncReceiveSpan(SYNC_RECEIVE_SPAN_NAME, startTime, toDecorate, Context.NONE); + }); + return new WindowedSubscriber<>(Collections.singletonMap(PARTITION_ID_KEY, partitionId), TERMINAL_MESSAGE, options); + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/AmqpReceiveLinkProcessor.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/AmqpReceiveLinkProcessor.java index f21a1a288b5f5..f6ea231c451d3 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/AmqpReceiveLinkProcessor.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/AmqpReceiveLinkProcessor.java @@ -13,6 +13,7 @@ import com.azure.core.util.AsyncCloseable; import com.azure.core.util.logging.ClientLogger; import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; +import com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationScope; import org.apache.qpid.proton.message.Message; import org.reactivestreams.Subscription; import reactor.core.CoreSubscriber; @@ -574,13 +575,11 @@ private void drainQueue() { return; } - Exception exception = null; - com.azure.core.util.Context span = instrumentation.asyncConsume("EventHubs.consume", message, partitionId, com.azure.core.util.Context.NONE); - AutoCloseable scope = instrumentation.getTracer().makeSpanCurrent(span); + InstrumentationScope scope = instrumentation.startAsyncConsume(message, partitionId); try { subscriber.onNext(message); } catch (Exception e) { - exception = e; + scope.setError(e); logger.atError() .addKeyValue(LINK_NAME_KEY, currentLinkName) .addKeyValue(ENTITY_PATH_KEY, entityPath) @@ -589,7 +588,7 @@ private void drainQueue() { throw logger.logExceptionAsError(Exceptions.propagate( Operators.onOperatorError(upstream, e, message, subscriber.currentContext()))); } finally { - instrumentation.getTracer().endSpan(exception, span, scope); + scope.close(); } numberEmitted++; diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/ClientConstants.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/ClientConstants.java index 5131f2058e85c..bbc10e214db5f 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/ClientConstants.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/ClientConstants.java @@ -42,5 +42,4 @@ public final class ClientConstants { * URI format for an Event Hubs FQDN. */ public static final String ENDPOINT_FORMAT = "sb://%s.%s"; - public static final String AZ_TRACING_SERVICE_NAME = "EventHubs."; } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java index e3a4304d29e86..af2ea5e673ebd 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java @@ -5,27 +5,36 @@ import com.azure.core.util.Context; import com.azure.core.util.metrics.Meter; -import com.azure.core.util.tracing.SpanKind; import com.azure.core.util.tracing.StartSpanOptions; import com.azure.core.util.tracing.Tracer; +import com.azure.messaging.eventhubs.EventData; import com.azure.messaging.eventhubs.implementation.MessageUtils; +import com.azure.messaging.eventhubs.models.EventBatchContext; +import com.azure.messaging.eventhubs.models.EventContext; +import com.azure.messaging.eventhubs.models.PartitionEvent; import org.apache.qpid.proton.amqp.Symbol; +import org.apache.qpid.proton.amqp.messaging.ApplicationProperties; import org.apache.qpid.proton.message.Message; +import reactor.core.publisher.Flux; import java.time.Instant; -import java.time.ZoneOffset; +import java.util.function.BiConsumer; import static com.azure.core.amqp.AmqpMessageConstant.ENQUEUED_TIME_UTC_ANNOTATION_NAME; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.MESSAGE_ENQUEUED_TIME_ATTRIBUTE_NAME; +import static com.azure.core.util.tracing.SpanKind.CONSUMER; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_BATCH_MESSAGE_COUNT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_PARTITION_ID; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.RECEIVE; public class EventHubsConsumerInstrumentation { private static final Symbol ENQUEUED_TIME_UTC_ANNOTATION_NAME_SYMBOL = Symbol.valueOf(ENQUEUED_TIME_UTC_ANNOTATION_NAME.getValue()); + private static final InstrumentationScope NOOP_SCOPE = new InstrumentationScope(null, null, null); private final EventHubsTracer tracer; private final EventHubsMetricsProvider meter; private final boolean isSync; public EventHubsConsumerInstrumentation(Tracer tracer, Meter meter, String fullyQualifiedName, String entityName, String consumerGroup, boolean isSyncConsumer) { - this.tracer = new EventHubsTracer(tracer, fullyQualifiedName, entityName); + this.tracer = new EventHubsTracer(tracer, fullyQualifiedName, entityName, consumerGroup); this.meter = new EventHubsMetricsProvider(meter, fullyQualifiedName, entityName, consumerGroup); this.isSync = isSyncConsumer; } @@ -34,26 +43,106 @@ public EventHubsTracer getTracer() { return tracer; } - public Context asyncConsume(String spanName, Message message, String partitionId, Context parent) { - if (!meter.isConsumerLagEnabled() && !tracer.isEnabled()) { - return parent; + public InstrumentationScope createScope(BiConsumer reportMetricsCallback) { + return isEnabled() ? new InstrumentationScope(tracer, meter, reportMetricsCallback) : NOOP_SCOPE; + } + + public InstrumentationScope startAsyncConsume(Message message, String partitionId) { + if (!isEnabled()) { + return NOOP_SCOPE; } + InstrumentationScope scope = createScope((m, s) -> { + if (!isSync) { + m.reportProcess(1, partitionId, s); + } + }); Instant enqueuedTime = MessageUtils.getEnqueuedTime(message.getMessageAnnotations().getValue(), ENQUEUED_TIME_UTC_ANNOTATION_NAME_SYMBOL); - Context child = parent; - if (tracer.isEnabled() && !isSync) { - StartSpanOptions options = tracer.createStartOption(SpanKind.CONSUMER, EventHubsTracer.OperationName.PROCESS) - .setAttribute(MESSAGE_ENQUEUED_TIME_ATTRIBUTE_NAME, enqueuedTime.atOffset(ZoneOffset.UTC).toEpochSecond()); + if (!isSync) { + ApplicationProperties properties = message.getApplicationProperties(); + scope.setSpan(tracer.startProcessSpan(properties == null ? null : properties.getValue(), + enqueuedTime, + partitionId, + Context.NONE)) + .makeSpanCurrent(); + } - if (message.getApplicationProperties() != null) { - options.setRemoteParent(tracer.extractContext(message.getApplicationProperties().getValue())); - } + if (enqueuedTime != null) { + meter.reportLag(enqueuedTime, partitionId, scope); + } + return scope; + } + + public Flux syncReceive(Flux events, String partitionId) { + if (!isEnabled()) { + return events; + } + + StartSpanOptions startOptions = tracer.isEnabled() ? tracer.createStartOptions(CONSUMER, RECEIVE, partitionId) : null; + Integer[] receivedCount = new Integer[]{0}; + + return Flux.using( + () -> { + if (startOptions != null) { + startOptions.setStartTimestamp(Instant.now()); + } - child = tracer.startSpan(spanName, options, parent); + return createScope((m, s) -> meter.reportReceive(receivedCount[0], partitionId, s)); + }, + scope -> events + .doOnNext(partitionEvent -> { + if (startOptions != null) { + receivedCount[0] = receivedCount[0] + 1; + EventData data = partitionEvent.getData(); + startOptions.addLink(tracer.createLink(data.getProperties(), data.getEnqueuedTime())); + } + }) + .doOnError(scope::setError) + .doOnCancel(scope::setCancelled), + scope -> { + if (startOptions != null) { + startOptions.setAttribute(MESSAGING_BATCH_MESSAGE_COUNT, receivedCount[0]); + startOptions.setAttribute(MESSAGING_DESTINATION_PARTITION_ID, partitionId); + + scope.setSpan(tracer.startSpan(RECEIVE, startOptions, Context.NONE)); + } + scope.close(); + }); + } + + public InstrumentationScope startProcess(EventBatchContext batchContext) { + if (batchContext.getEvents().isEmpty() || !isEnabled()) { + return NOOP_SCOPE; } - meter.reportReceive(enqueuedTime, partitionId, child); + InstrumentationScope scope = createScope((m, s) -> + m.reportProcess(batchContext.getEvents().size(), batchContext.getPartitionContext().getPartitionId(), s)); + + return scope + .setSpan(tracer.startProcessSpan(batchContext, Context.NONE)) + .makeSpanCurrent(); + } + + public InstrumentationScope startProcess(EventContext eventContext) { + EventData event = eventContext.getEventData(); + if (event == null || !isEnabled()) { + return NOOP_SCOPE; + } + + InstrumentationScope scope = createScope((m, s) -> + m.reportProcess(1, eventContext.getPartitionContext().getPartitionId(), s)); + + Context span = tracer.startProcessSpan(event.getProperties(), + event.getEnqueuedTime(), + eventContext.getPartitionContext().getPartitionId(), + Context.NONE); + + return scope + .setSpan(span) + .makeSpanCurrent(); + } - return child; + boolean isEnabled() { + return tracer.isEnabled() || meter.isEnabled(); } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java index 38f1813ab7ea7..c61cde3aad20d 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java @@ -3,94 +3,193 @@ package com.azure.messaging.eventhubs.implementation.instrumentation; -import com.azure.core.util.Context; import com.azure.core.util.TelemetryAttributes; +import com.azure.core.util.logging.ClientLogger; import com.azure.core.util.metrics.DoubleHistogram; import com.azure.core.util.metrics.LongCounter; import com.azure.core.util.metrics.Meter; +import com.azure.messaging.eventhubs.models.Checkpoint; import java.time.Instant; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static com.azure.core.amqp.implementation.ClientConstants.ENTITY_NAME_KEY; -import static com.azure.core.amqp.implementation.ClientConstants.HOSTNAME_KEY; -import static com.azure.messaging.eventhubs.implementation.ClientConstants.CONSUMER_GROUP_KEY; -import static com.azure.messaging.eventhubs.implementation.ClientConstants.PARTITION_ID_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.ERROR_TYPE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_CLIENT_CONSUMED_MESSAGES; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_CLIENT_OPERATION_DURATION; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_CLIENT_PUBLISHED_MESSAGES; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_CONSUMER_GROUP_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_EVENTHUBS_CONSUMER_LAG; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_PARTITION_ID; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_TYPE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_PROCESS_DURATION; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_SYSTEM; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_SYSTEM_VALUE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.SERVER_ADDRESS; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.getDurationInSeconds; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.getOperationType; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.CHECKPOINT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.PROCESS; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.RECEIVE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.SEND; public class EventHubsMetricsProvider { - private static final String GENERIC_STATUS_KEY = "status"; private final Meter meter; private final boolean isEnabled; - + private static final ClientLogger LOGGER = new ClientLogger(EventHubsMetricsProvider.class); + private Map commonAttributes; private AttributeCache sendAttributeCacheSuccess; - private AttributeCache sendAttributeCacheFailure; - private AttributeCache receiveAttributeCache; - private LongCounter sentEventsCounter; + private AttributeCache receiveAttributeCacheSuccess; + private AttributeCache checkpointAttributeCacheSuccess; + private AttributeCache processAttributeCacheSuccess; + private AttributeCache lagAttributeCache; + private LongCounter publishedEventCounter; + private LongCounter consumedEventCounter; + private DoubleHistogram operationDuration; + private DoubleHistogram processDuration; private DoubleHistogram consumerLag; - public EventHubsMetricsProvider(Meter meter, String namespace, String entityName, String consumerGroup) { this.meter = meter; this.isEnabled = meter != null && meter.isEnabled(); if (this.isEnabled) { - Map commonAttributesMap = new HashMap<>(3); - commonAttributesMap.put(HOSTNAME_KEY, namespace); - commonAttributesMap.put(ENTITY_NAME_KEY, entityName); - if (consumerGroup != null) { - commonAttributesMap.put(CONSUMER_GROUP_KEY, consumerGroup); - } + this.commonAttributes = getCommonAttributes(namespace, entityName, consumerGroup); + this.sendAttributeCacheSuccess = AttributeCache.create(meter, SEND, commonAttributes); + this.receiveAttributeCacheSuccess = AttributeCache.create(meter, RECEIVE, commonAttributes); + this.checkpointAttributeCacheSuccess = AttributeCache.create(meter, CHECKPOINT, commonAttributes); + this.processAttributeCacheSuccess = AttributeCache.create(meter, PROCESS, commonAttributes); + this.lagAttributeCache = new AttributeCache(meter, MESSAGING_DESTINATION_PARTITION_ID, commonAttributes); - this.sendAttributeCacheSuccess = new AttributeCache(PARTITION_ID_KEY, new HashMap<>(commonAttributesMap)); + this.publishedEventCounter = meter.createLongCounter(MESSAGING_CLIENT_PUBLISHED_MESSAGES, "The number of published events", "{event}"); + this.consumedEventCounter = meter.createLongCounter(MESSAGING_CLIENT_CONSUMED_MESSAGES, "The number of consumed events", "{event}"); - Map failureMap = new HashMap<>(commonAttributesMap); - failureMap.put(GENERIC_STATUS_KEY, "error"); - this.sendAttributeCacheFailure = new AttributeCache(PARTITION_ID_KEY, failureMap); + this.operationDuration = meter.createDoubleHistogram(MESSAGING_CLIENT_OPERATION_DURATION, "The duration of client messaging operations involving communication with the Event Hubs namespace", "s"); + this.processDuration = meter.createDoubleHistogram(MESSAGING_PROCESS_DURATION, "The duration of the processing callback", "s"); - this.receiveAttributeCache = new AttributeCache(PARTITION_ID_KEY, commonAttributesMap); - this.sentEventsCounter = meter.createLongCounter("messaging.eventhubs.events.sent", "Number of sent events", "events"); - this.consumerLag = meter.createDoubleHistogram("messaging.eventhubs.consumer.lag", "Difference between local time when event was received and the local time it was enqueued on broker.", "sec"); + this.consumerLag = meter.createDoubleHistogram(MESSAGING_EVENTHUBS_CONSUMER_LAG, "Difference between local time when event was received and the local time it was enqueued on broker", "s"); } } - public boolean isSendCountEnabled() { - return isEnabled && sentEventsCounter.isEnabled(); + public boolean isEnabled() { + return isEnabled; } - public void reportBatchSend(int batchSize, String partitionId, Throwable throwable, Context context) { - if (isEnabled && sentEventsCounter.isEnabled()) { - AttributeCache cache = throwable == null ? sendAttributeCacheSuccess : sendAttributeCacheFailure; - sentEventsCounter.add(batchSize, cache.getOrCreate(partitionId), context); + public void reportBatchSend(int batchSize, String partitionId, InstrumentationScope scope) { + if (isEnabled && (publishedEventCounter.isEnabled() || operationDuration.isEnabled())) { + TelemetryAttributes attributes = getOrCreateAttributes(SEND, partitionId, scope.getErrorType()); + publishedEventCounter.add(batchSize, attributes, scope.getSpan()); + operationDuration.record(getDurationInSeconds(scope.getStartTime()), attributes, scope.getSpan()); } } - public boolean isConsumerLagEnabled() { - return isEnabled && consumerLag.isEnabled(); + public void reportProcess(int batchSize, String partitionId, InstrumentationScope scope) { + if (isEnabled && (consumedEventCounter.isEnabled() || processDuration.isEnabled())) { + TelemetryAttributes attributes = getOrCreateAttributes(PROCESS, partitionId, scope.getErrorType()); + consumedEventCounter.add(batchSize, attributes, scope.getSpan()); + processDuration.record(getDurationInSeconds(scope.getStartTime()), attributes, scope.getSpan()); + } + } + + public void reportReceive(int receivedCount, String partitionId, InstrumentationScope scope) { + if (isEnabled && (operationDuration.isEnabled() || consumedEventCounter.isEnabled())) { + String errorType = scope.getErrorType(); + TelemetryAttributes attributes = getOrCreateAttributes(RECEIVE, partitionId, errorType); + if (receivedCount > 0) { + consumedEventCounter.add(receivedCount, + errorType == null ? attributes : getOrCreateAttributes(RECEIVE, partitionId, null), + scope.getSpan()); + } + + operationDuration.record(getDurationInSeconds(scope.getStartTime()), attributes, scope.getSpan()); + } } - public void reportReceive(Instant enqueuedTime, String partitionId, Context context) { + public void reportLag(Instant enqueuedTime, String partitionId, InstrumentationScope scope) { if (isEnabled && consumerLag.isEnabled()) { - double diff = 0d; - if (enqueuedTime != null) { - diff = Instant.now().toEpochMilli() - enqueuedTime.toEpochMilli(); - if (diff < 0) { - // time skew on machines - diff = 0; - } + consumerLag.record(getDurationInSeconds(enqueuedTime), lagAttributeCache.getOrCreate(partitionId), scope.getSpan()); + } + } + + public void reportCheckpoint(Checkpoint checkpoint, InstrumentationScope scope) { + if (isEnabled && operationDuration.isEnabled()) { + operationDuration.record(getDurationInSeconds(scope.getStartTime()), + getOrCreateAttributes(CHECKPOINT, checkpoint.getPartitionId(), scope.getErrorType()), scope.getSpan()); + } + } + + private TelemetryAttributes getOrCreateAttributes(OperationName operationName, String partitionId, String errorType) { + if (errorType == null) { + switch (operationName) { + case SEND: + return sendAttributeCacheSuccess.getOrCreate(partitionId); + case RECEIVE: + return receiveAttributeCacheSuccess.getOrCreate(partitionId); + case CHECKPOINT: + return checkpointAttributeCacheSuccess.getOrCreate(partitionId); + case PROCESS: + return processAttributeCacheSuccess.getOrCreate(partitionId); + default: + LOGGER.atVerbose() + .addKeyValue("operationName", operationName) + .log("Unknown operation name"); + // this should never happen + return lagAttributeCache.getOrCreate(partitionId); } - consumerLag.record(diff / 1000d, receiveAttributeCache.getOrCreate(partitionId), context); } + + // we can potentially cache failure attributes, but the assumption is that + // the cost of creating attributes is negligible comparing to throwing an exception + Map attributes = new HashMap<>(commonAttributes); + if (partitionId != null) { + attributes.put(MESSAGING_DESTINATION_PARTITION_ID, partitionId); + } + + setOperation(attributes, operationName); + attributes.put(ERROR_TYPE, errorType); + return meter.createAttributes(attributes); + } + + private Map getCommonAttributes(String namespace, String entityName, String consumerGroup) { + Map commonAttributesMap = new HashMap<>(3); + commonAttributesMap.put(MESSAGING_SYSTEM, MESSAGING_SYSTEM_VALUE); + commonAttributesMap.put(SERVER_ADDRESS, namespace); + commonAttributesMap.put(MESSAGING_DESTINATION_NAME, entityName); + if (consumerGroup != null) { + commonAttributesMap.put(MESSAGING_CONSUMER_GROUP_NAME, consumerGroup); + } + + return Collections.unmodifiableMap(commonAttributesMap); } - class AttributeCache { + private static void setOperation(Map attributes, OperationName name) { + String operationType = getOperationType(name); + if (operationType != null) { + attributes.put(MESSAGING_OPERATION_TYPE, operationType); + } + + attributes.put(MESSAGING_OPERATION_NAME, name.toString()); + } + + private static final class AttributeCache { private final Map attr = new ConcurrentHashMap<>(); private final TelemetryAttributes commonAttr; private final Map commonMap; private final String dimensionName; + private final Meter meter; + + static AttributeCache create(Meter meter, OperationName operationName, Map commonAttributes) { + Map attributes = new HashMap<>(commonAttributes); + setOperation(attributes, operationName); + return new AttributeCache(meter, MESSAGING_DESTINATION_PARTITION_ID, attributes); + } - AttributeCache(String dimensionName, Map common) { + private AttributeCache(Meter meter, String dimensionName, Map common) { this.dimensionName = dimensionName; this.commonMap = common; + this.meter = meter; this.commonAttr = meter.createAttributes(commonMap); } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java index 9d6fba525e39b..a102d3e0a5683 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java @@ -3,101 +3,91 @@ package com.azure.messaging.eventhubs.implementation.instrumentation; -import com.azure.core.amqp.exception.AmqpException; import com.azure.core.util.Configuration; import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; import com.azure.core.util.logging.ClientLogger; import com.azure.core.util.tracing.SpanKind; import com.azure.core.util.tracing.StartSpanOptions; import com.azure.core.util.tracing.Tracer; import com.azure.core.util.tracing.TracingLink; import com.azure.messaging.eventhubs.EventData; -import com.azure.messaging.eventhubs.models.PartitionEvent; -import reactor.core.publisher.Flux; +import com.azure.messaging.eventhubs.models.EventBatchContext; import reactor.core.publisher.Mono; import java.time.Instant; import java.time.ZoneOffset; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import static com.azure.core.util.tracing.Tracer.ENTITY_PATH_KEY; -import static com.azure.core.util.tracing.Tracer.HOST_NAME_KEY; +import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; import static com.azure.core.util.tracing.Tracer.SPAN_CONTEXT_KEY; +import static com.azure.messaging.eventhubs.EventHubClientBuilder.DEFAULT_CONSUMER_GROUP_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.DIAGNOSTIC_ID_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_BATCH_MESSAGE_COUNT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_CONSUMER_GROUP_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_PARTITION_ID; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_TYPE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_SYSTEM; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_SYSTEM_VALUE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.SERVER_ADDRESS; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.TRACEPARENT_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.getErrorType; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.getOperationType; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.unwrap; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.EVENT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.PROCESS; public class EventHubsTracer { private static final AutoCloseable NOOP_AUTOCLOSEABLE = () -> { }; - public static final String REACTOR_PARENT_TRACE_CONTEXT_KEY = "otel-context-key"; - public static final String TRACEPARENT_KEY = "traceparent"; - public static final String DIAGNOSTIC_ID_KEY = "Diagnostic-Id"; - public static final String MESSAGE_ENQUEUED_TIME_ATTRIBUTE_NAME = "messaging.eventhubs.message.enqueued_time"; - public static final String MESSAGING_BATCH_SIZE_ATTRIBUTE_NAME = "messaging.batch.message_count"; - private static final String MESSAGING_SYSTEM_ATTRIBUTE_NAME = "messaging.system"; - private static final String MESSAGING_OPERATION_ATTRIBUTE_NAME = "messaging.operation"; private static final TracingLink DUMMY_LINK = new TracingLink(Context.NONE); private static final ClientLogger LOGGER = new ClientLogger(EventHubsTracer.class); private static final boolean IS_TRACING_DISABLED = Configuration.getGlobalConfiguration().get(Configuration.PROPERTY_AZURE_TRACING_DISABLED, false); + protected final Tracer tracer; private final String fullyQualifiedName; private final String entityName; + private final String consumerGroup; - public EventHubsTracer(Tracer tracer, String fullyQualifiedName, String entityName) { + public EventHubsTracer(Tracer tracer, String fullyQualifiedName, String entityName, String consumerGroup) { this.tracer = IS_TRACING_DISABLED ? null : tracer; this.fullyQualifiedName = Objects.requireNonNull(fullyQualifiedName, "'fullyQualifiedName' cannot be null"); this.entityName = Objects.requireNonNull(entityName, "'entityPath' cannot be null"); + this.consumerGroup = DEFAULT_CONSUMER_GROUP_NAME.equalsIgnoreCase(consumerGroup) ? null : consumerGroup; } public boolean isEnabled() { return tracer != null && tracer.isEnabled(); } - public Context startSpan(String spanName, StartSpanOptions startOptions, Context context) { - return isEnabled() ? tracer.start(spanName, startOptions, context) : context; + public Context startSpan(OperationName operationName, StartSpanOptions startOptions, Context context) { + return isEnabled() ? tracer.start(getSpanName(operationName), startOptions, context) : context; } - public Mono traceMono(Mono publisher, String spanName) { - if (isEnabled()) { - return publisher - .doOnEach(signal -> { - if (signal.isOnComplete() || signal.isOnError()) { - Context span = signal.getContextView().getOrDefault(REACTOR_PARENT_TRACE_CONTEXT_KEY, Context.NONE); - endSpan(signal.getThrowable(), span, null); - } - }) - .contextWrite(reactor.util.context.Context.of(REACTOR_PARENT_TRACE_CONTEXT_KEY, - tracer.start(spanName, createStartOption(SpanKind.CLIENT, null), Context.NONE))); + public Mono traceMono(Mono publisher, OperationName operationName, String partitionId) { + if (!isEnabled()) { + return publisher; } - return publisher; - } - - public void endSpan(Throwable throwable, Context span, AutoCloseable scope) { - if (isEnabled()) { - String errorCondition = null; - if (throwable instanceof AmqpException) { - AmqpException exception = (AmqpException) throwable; - if (exception.getErrorCondition() != null) { - errorCondition = exception.getErrorCondition().getErrorCondition(); - } - } - - try { - if (scope != null) { - scope.close(); - } - } catch (Exception e) { - LOGGER.warning("Can't close scope", e); - } finally { - tracer.end(errorCondition, throwable, span); - } - } + return Mono.using( + () -> new InstrumentationScope(this, null, null) + .setSpan(tracer.start(getSpanName(operationName), + createStartOptions(SpanKind.CLIENT, operationName, partitionId), + Context.NONE)), + scope -> publisher + .doOnError(scope::setError) + .doOnCancel(scope::setCancelled) + .contextWrite(c -> c.put(PARENT_TRACE_CONTEXT_KEY, scope.getSpan())), + InstrumentationScope::close); } /** @@ -116,9 +106,9 @@ public void reportMessageSpan(EventData eventData, Context eventContext) { } // Starting the span makes the sampling decision (nothing is logged at this time) - StartSpanOptions startOptions = createStartOption(SpanKind.PRODUCER, null); + StartSpanOptions startOptions = createStartOptions(SpanKind.PRODUCER, EVENT, null); - Context eventSpanContext = tracer.start("EventHubs.message", startOptions, eventContext); + Context eventSpanContext = tracer.start(getSpanName(EVENT), startOptions, eventContext); Exception exception = null; String error = null; @@ -148,11 +138,20 @@ public void reportMessageSpan(EventData eventData, Context eventContext) { } } - private static boolean canModifyApplicationProperties(Map applicationProperties) { - return applicationProperties != null && !applicationProperties.getClass().getSimpleName().equals("UnmodifiableMap"); + public TracingLink createLink(Map applicationProperties, Instant enqueuedTime) { + return createLink(extractContext(applicationProperties), enqueuedTime); + } + + private TracingLink createLink(Context linkContext, Instant enqueuedTime) { + Map linkAttributes = null; + if (enqueuedTime != null) { + linkAttributes = Collections.singletonMap(MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME, enqueuedTime.atOffset(ZoneOffset.UTC).toEpochSecond()); + } + + return new TracingLink(linkContext, linkAttributes); } - public TracingLink createLink(Map applicationProperties, Instant enqueuedTime, Context eventContext) { + public TracingLink createProducerLink(Map applicationProperties, Context eventContext) { if (!tracer.isEnabled() || applicationProperties == null) { return DUMMY_LINK; } @@ -169,88 +168,82 @@ public TracingLink createLink(Map applicationProperties, Instant link = extractContext(applicationProperties); } - Map linkAttributes = null; - if (enqueuedTime != null) { - linkAttributes = Collections.singletonMap(MESSAGE_ENQUEUED_TIME_ATTRIBUTE_NAME, enqueuedTime.atOffset(ZoneOffset.UTC).toEpochSecond()); - } - - return new TracingLink(link, linkAttributes); + return new TracingLink(link, null); } - public Context extractContext(Map applicationProperties) { - if (tracer.isEnabled() && applicationProperties != null) { - return tracer.extractContext(key -> { - if (TRACEPARENT_KEY.equals(key)) { - return getTraceparent(applicationProperties); - } else { - Object value = applicationProperties.get(key); - if (value != null) { - return value.toString(); - } + void endSpan(String errorType, Throwable throwable, Context context, AutoCloseable scope) { + if (isEnabled()) { + try { + if (scope != null) { + scope.close(); } - return null; - }); + } catch (Exception e) { + LOGGER.verbose("Can't close scope", e); + } finally { + tracer.end(errorType == null ? getErrorType(throwable) : errorType, unwrap(throwable), context); + } } - - return Context.NONE; } - public AutoCloseable makeSpanCurrent(Context context) { + AutoCloseable makeSpanCurrent(Context context) { return isEnabled() ? tracer.makeSpanCurrent(context) : NOOP_AUTOCLOSEABLE; } - public Context startProcessSpan(String name, EventData event, Context parent) { - if (isEnabled() && event != null) { - StartSpanOptions startOptions = createStartOption(SpanKind.CONSUMER, OperationName.PROCESS) - .setRemoteParent(extractContext(event.getProperties())); + Context startProcessSpan(Map applicationProperties, Instant enqueuedTime, String partitionId, Context parent) { + if (isEnabled()) { + Context remoteContext = extractContext(applicationProperties); + StartSpanOptions startOptions = createStartOptions(SpanKind.CONSUMER, PROCESS, partitionId) + .addLink(createLink(remoteContext, null)) + .setRemoteParent(remoteContext); - Instant enqueuedTime = event.getEnqueuedTime(); + // we could add these attributes only on sampled-in spans, but we don't have the API to check if (enqueuedTime != null) { - startOptions.setAttribute(MESSAGE_ENQUEUED_TIME_ATTRIBUTE_NAME, enqueuedTime.atOffset(ZoneOffset.UTC).toEpochSecond()); + startOptions.setAttribute(MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME, enqueuedTime.atOffset(ZoneOffset.UTC).toEpochSecond()); } - return tracer.start(name, startOptions, parent); + return tracer.start(getSpanName(PROCESS), startOptions, parent); } return parent; } - public Context startProcessSpan(String name, List events, Context parent) { - if (isEnabled() && events != null) { - StartSpanOptions startOptions = createStartOption(SpanKind.CONSUMER, OperationName.PROCESS); - startOptions.setAttribute(MESSAGING_BATCH_SIZE_ATTRIBUTE_NAME, events.size()); - for (EventData event : events) { - startOptions.addLink(createLink(event.getProperties(), event.getEnqueuedTime(), Context.NONE)); + Context startProcessSpan(EventBatchContext batchContext, Context parent) { + if (isEnabled() && batchContext != null && !CoreUtils.isNullOrEmpty(batchContext.getEvents())) { + StartSpanOptions startOptions = createStartOptions(SpanKind.CONSUMER, PROCESS, + batchContext.getPartitionContext().getPartitionId()); + startOptions.setAttribute(MESSAGING_BATCH_MESSAGE_COUNT, batchContext.getEvents().size()); + + for (EventData event : batchContext.getEvents()) { + startOptions.addLink(createLink(event.getProperties(), event.getEnqueuedTime())); } - return tracer.start(name, startOptions, parent); + return tracer.start(getSpanName(PROCESS), startOptions, parent); } return parent; } - public Flux reportSyncReceiveSpan(String name, Instant startTime, Flux events, Context parent) { - if (isEnabled() && events != null) { - final StartSpanOptions startOptions = createStartOption(SpanKind.CLIENT, OperationName.RECEIVE) - .setStartTimestamp(startTime); + public StartSpanOptions createStartOptions(SpanKind kind, OperationName operationName, String partitionId) { + StartSpanOptions startOptions = new StartSpanOptions(kind) + .setAttribute(MESSAGING_SYSTEM, MESSAGING_SYSTEM_VALUE) + .setAttribute(MESSAGING_DESTINATION_NAME, entityName) + .setAttribute(SERVER_ADDRESS, fullyQualifiedName) + .setAttribute(MESSAGING_OPERATION_NAME, operationName.toString()); - return events.doOnEach(signal -> { - if (signal.hasValue()) { - EventData data = signal.get().getData(); - if (data != null) { - startOptions.addLink(createLink(data.getProperties(), data.getEnqueuedTime(), Context.NONE)); - } - } else if (signal.isOnComplete() || signal.isOnError()) { - int batchSize = startOptions.getLinks() == null ? 0 : startOptions.getLinks().size(); - startOptions.setAttribute(MESSAGING_BATCH_SIZE_ATTRIBUTE_NAME, batchSize); + if (consumerGroup != null) { + startOptions.setAttribute(MESSAGING_CONSUMER_GROUP_NAME, consumerGroup); + } - Context span = tracer.start(name, startOptions, parent); - tracer.end(null, signal.getThrowable(), span); - } - }); + if (partitionId != null) { + startOptions.setAttribute(MESSAGING_DESTINATION_PARTITION_ID, partitionId); } - return events; + String operationType = getOperationType(operationName); + if (operationType != null) { + startOptions.setAttribute(MESSAGING_OPERATION_TYPE, operationType); + } + + return startOptions; } private static String getTraceparent(Map applicationProperties) { @@ -262,32 +255,29 @@ private static String getTraceparent(Map applicationProperties) return diagnosticId == null ? null : diagnosticId.toString(); } - public StartSpanOptions createStartOption(SpanKind kind, OperationName operationName) { - StartSpanOptions startOptions = new StartSpanOptions(kind) - .setAttribute(MESSAGING_SYSTEM_ATTRIBUTE_NAME, "eventhubs") - .setAttribute(ENTITY_PATH_KEY, entityName) - .setAttribute(HOST_NAME_KEY, fullyQualifiedName); - - if (operationName != null) { - startOptions.setAttribute(MESSAGING_OPERATION_ATTRIBUTE_NAME, operationName.toString()); + private Context extractContext(Map applicationProperties) { + if (tracer.isEnabled() && applicationProperties != null) { + return tracer.extractContext(key -> { + if (TRACEPARENT_KEY.equals(key)) { + return getTraceparent(applicationProperties); + } else { + Object value = applicationProperties.get(key); + if (value != null) { + return value.toString(); + } + } + return null; + }); } - return startOptions; + return Context.NONE; } - public enum OperationName { - PUBLISH("publish"), - RECEIVE("receive"), - PROCESS("process"); - - private final String operationName; - OperationName(String operationName) { - this.operationName = operationName; - } + private String getSpanName(OperationName operationName) { + return operationName.toString() + " " + entityName; + } - @Override - public String toString() { - return operationName; - } + private static boolean canModifyApplicationProperties(Map applicationProperties) { + return applicationProperties != null && !applicationProperties.getClass().getSimpleName().equals("UnmodifiableMap"); } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java new file mode 100644 index 0000000000000..b8132540d210e --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs.implementation.instrumentation; + +import com.azure.core.util.Context; + +import java.time.Instant; +import java.util.function.BiConsumer; + +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.CANCELLED_ERROR_TYPE_VALUE; + +/** + * Helper class that holds state necessary for the operation instrumentation. + */ +public final class InstrumentationScope implements AutoCloseable { + private final EventHubsMetricsProvider meter; + private final EventHubsTracer tracer; + private final boolean isEnabled; + private Instant startTime; + private Throwable error; + private String errorType; + private Context span = Context.NONE; + private AutoCloseable spanScope; + private BiConsumer reportMetricsCallback; + private boolean closed = false; + + public InstrumentationScope(EventHubsTracer tracer, + EventHubsMetricsProvider meter, + BiConsumer reportMetricsCallback) { + this.tracer = tracer; + this.meter = meter; + this.isEnabled = (tracer != null && tracer.isEnabled()) || (meter != null && meter.isEnabled()); + if (meter != null && meter.isEnabled()) { // micro-optimization + this.startTime = Instant.now(); + } + this.reportMetricsCallback = reportMetricsCallback; + } + + public boolean isEnabled() { + return isEnabled; + } + + public InstrumentationScope setError(Throwable error) { + if (isEnabled) { + this.error = error; + } + return this; + } + + public InstrumentationScope setSpan(Context span) { + if (isEnabled) { + this.span = span; + } + return this; + } + + public InstrumentationScope setCancelled() { + // Complicated calls can result in error followed by cancellation. We shouldn't track them twice. + // don't trust me? try this: + // Flux.fromIterable(Collections.singletonList("event")) + // .flatMap(event -> Mono.error(new RuntimeException("boom")) + // .doOnError(e -> System.out.println("Error")) + // .doOnCancel(() -> System.out.println("Cancel"))) + // .blockLast(); + if (isEnabled && error == null) { + errorType = CANCELLED_ERROR_TYPE_VALUE; + } + return this; + } + + public Instant getStartTime() { + return startTime; + } + + public Context getSpan() { + return span; + } + + public Throwable getError() { + return error; + } + + public String getErrorType() { + if (errorType == null) { + errorType = InstrumentationUtils.getErrorType(error); + } + return errorType; + } + + @Override + public void close() { + if (!closed) { + closed = true; + if (meter != null && reportMetricsCallback != null) { + reportMetricsCallback.accept(meter, this); + } + if (tracer != null) { + tracer.endSpan(errorType, error, span, spanScope); + } + } + } + + InstrumentationScope makeSpanCurrent() { + if (tracer != null && tracer.isEnabled()) { + spanScope = tracer.makeSpanCurrent(span); + } + return this; + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java new file mode 100644 index 0000000000000..e4ea48bd84702 --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs.implementation.instrumentation; + +import com.azure.core.amqp.exception.AmqpException; +import reactor.core.Exceptions; +import reactor.core.publisher.Signal; + +import java.time.Duration; +import java.time.Instant; + +public final class InstrumentationUtils { + // Attribute names based on OpenTelemetry specification + // https://github.com/open-telemetry/semantic-conventions/blob/main/docs/messaging/messaging-spans.md + public static final String SERVER_ADDRESS = "server.address"; + public static final String ERROR_TYPE = "error.type"; + public static final String MESSAGING_BATCH_MESSAGE_COUNT = "messaging.batch.message_count"; + public static final String MESSAGING_DESTINATION_NAME = "messaging.destination.name"; + public static final String MESSAGING_DESTINATION_PARTITION_ID = "messaging.destination.partition.id"; + public static final String MESSAGING_OPERATION_NAME = "messaging.operation.name"; + public static final String MESSAGING_OPERATION_TYPE = "messaging.operation.type"; + public static final String MESSAGING_SYSTEM = "messaging.system"; + public static final String MESSAGING_CONSUMER_GROUP_NAME = "messaging.consumer.group.name"; + public static final String MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME = "messaging.eventhubs.message.enqueued_time"; + + // metrics + public static final String MESSAGING_CLIENT_PUBLISHED_MESSAGES = "messaging.client.published.messages"; + public static final String MESSAGING_CLIENT_CONSUMED_MESSAGES = "messaging.client.consumed.messages"; + public static final String MESSAGING_CLIENT_OPERATION_DURATION = "messaging.client.operation.duration"; + public static final String MESSAGING_PROCESS_DURATION = "messaging.process.duration"; + + // custom metrics + public static final String MESSAGING_EVENTHUBS_CONSUMER_LAG = "messaging.eventhubs.consumer.lag"; + + // constant attribute values + public static final String MESSAGING_SYSTEM_VALUE = "eventhubs"; + public static final String CANCELLED_ERROR_TYPE_VALUE = "cancelled"; + + // _OTHER is a magic string defined in OpenTelemetry for 'unknown' errors + public static final String OTHER_ERROR_TYPE_VALUE = "_OTHER"; + + // context propagation constants + public static final String TRACEPARENT_KEY = "traceparent"; + public static final String DIAGNOSTIC_ID_KEY = "Diagnostic-Id"; + + public static String getErrorType(Signal signal) { + // this method should only be called for complete or error signals + if (signal.isOnComplete()) { + return null; + } + + return getErrorType(signal.getThrowable()); + } + + public static String getErrorType(Throwable error) { + if (error == null) { + return null; + } + + error = Exceptions.unwrap(error); + + if (error instanceof AmqpException && ((AmqpException) error).getErrorCondition() != null) { + return ((AmqpException) error).getErrorCondition().getErrorCondition(); + } + + return (error != null) ? error.getClass().getName() : OTHER_ERROR_TYPE_VALUE; + } + + public static Throwable unwrap(Throwable error) { + error = Exceptions.unwrap(error); + + if (error instanceof AmqpException && error.getCause() != null) { + return error.getCause(); + } + + return error; + } + + public static double getDurationInSeconds(Instant startTime) { + long durationNanos = Duration.between(startTime, Instant.now()).toNanos(); + if (durationNanos < 0) { + // we use this method to get lag, so need to take care of time skew on different machines + return 0d; + } + return durationNanos / 1_000_000_000d; + } + + public static String getOperationType(OperationName name) { + switch (name) { + case SEND: + return "publish"; + case RECEIVE: + return "receive"; + case CHECKPOINT: + return "settle"; + case PROCESS: + return "process"; + default: + return null; + } + } + + private InstrumentationUtils() { + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedCheckpointStore.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedCheckpointStore.java new file mode 100644 index 0000000000000..0685847195287 --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedCheckpointStore.java @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs.implementation.instrumentation; + +import com.azure.core.util.Context; +import com.azure.core.util.tracing.SpanKind; +import com.azure.messaging.eventhubs.CheckpointStore; +import com.azure.messaging.eventhubs.models.Checkpoint; +import com.azure.messaging.eventhubs.models.PartitionOwnership; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.List; + +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.CHECKPOINT; + +public final class InstrumentedCheckpointStore implements CheckpointStore { + private final CheckpointStore checkpointStore; + private final EventHubsConsumerInstrumentation instrumentation; + private final EventHubsTracer tracer; + private InstrumentedCheckpointStore(CheckpointStore checkpointStore, EventHubsConsumerInstrumentation instrumentation) { + this.checkpointStore = checkpointStore; + this.instrumentation = instrumentation; + this.tracer = instrumentation.getTracer(); + } + + public static CheckpointStore create(CheckpointStore checkpointStore, EventHubsConsumerInstrumentation instrumentation) { + if (!instrumentation.isEnabled()) { + return checkpointStore; + } + + return new InstrumentedCheckpointStore(checkpointStore, instrumentation); + } + + @Override + public Flux listOwnership(String fullyQualifiedNamespace, String eventHubName, String consumerGroup) { + return checkpointStore.listOwnership(fullyQualifiedNamespace, eventHubName, consumerGroup); + } + + @Override + public Flux claimOwnership(List requestedPartitionOwnerships) { + return checkpointStore.claimOwnership(requestedPartitionOwnerships); + } + + @Override + public Flux listCheckpoints(String fullyQualifiedNamespace, String eventHubName, String consumerGroup) { + return checkpointStore.listCheckpoints(fullyQualifiedNamespace, eventHubName, consumerGroup); + } + + @Override + public Mono updateCheckpoint(Checkpoint checkpoint) { + return Mono.using( + () -> instrumentation.createScope((m, s) -> m.reportCheckpoint(checkpoint, s)) + .setSpan(startSpan(checkpoint.getPartitionId())), + scope -> checkpointStore.updateCheckpoint(checkpoint) + .doOnError(scope::setError) + .doOnCancel(scope::setCancelled) + .contextWrite(ctx -> ctx.putAllMap(scope.getSpan().getValues())), + InstrumentationScope::close); + } + + private Context startSpan(String partitionId) { + return tracer.isEnabled() + ? tracer.startSpan(CHECKPOINT, tracer.createStartOptions(SpanKind.INTERNAL, CHECKPOINT, partitionId), Context.NONE) + : Context.NONE; + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/OperationName.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/OperationName.java new file mode 100644 index 0000000000000..637f6476ae689 --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/OperationName.java @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs.implementation.instrumentation; + +public enum OperationName { + EVENT("event"), + PROCESS("process"), + SEND("send"), + RECEIVE("receive"), + CHECKPOINT("checkpoint"), + GET_EVENT_HUB_PROPERTIES("get_event_hub_properties"), + GET_PARTITION_PROPERTIES("get_partition_properties"); + + private final String value; + OperationName(String operationName) { + this.value = operationName; + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return this.value; + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/module-info.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/module-info.java index 0386dee075a72..20f02469f0c2e 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/module-info.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/module-info.java @@ -14,6 +14,6 @@ opens com.azure.messaging.eventhubs.models; opens com.azure.messaging.eventhubs.implementation; - + opens com.azure.messaging.eventhubs.implementation.instrumentation to com.azure.core; uses com.azure.core.util.tracing.Tracer; } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClientTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClientTest.java index 3dd78359de839..25857adda3e4a 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClientTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClientTest.java @@ -67,18 +67,19 @@ import java.time.OffsetDateTime; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static com.azure.core.amqp.AmqpMessageConstant.ENQUEUED_TIME_UTC_ANNOTATION_NAME; -import static com.azure.core.util.tracing.Tracer.ENTITY_PATH_KEY; -import static com.azure.core.util.tracing.Tracer.HOST_NAME_KEY; import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; import static com.azure.messaging.eventhubs.EventHubClientBuilder.DEFAULT_PREFETCH_COUNT; +import static com.azure.messaging.eventhubs.TestUtils.assertAllAttributes; import static com.azure.messaging.eventhubs.TestUtils.getMessage; +import static com.azure.messaging.eventhubs.TestUtils.getSpanName; import static com.azure.messaging.eventhubs.TestUtils.isMatchingEvent; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_EVENT_HUB_PROPERTIES; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_PARTITION_PROPERTIES; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -755,7 +756,8 @@ void receiveReportsMetrics() { List> measurements = consumerLag.getMeasurements(); TestMeasurement last = measurements.get(measurements.size() - 1); assertEquals(Duration.between(enqueuedTime, afterReceived).toMillis() / 1000d, last.getValue(), 1); - assertAttributes(e.getPartitionContext().getPartitionId(), last.getAttributes()); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, e.getPartitionContext().getPartitionId(), CONSUMER_GROUP, null, + null, last.getAttributes()); return true; }) .expectComplete() @@ -792,7 +794,8 @@ void receiveReportsMetricsNegativeLag() { List> measurements = consumerLag.getMeasurements(); TestMeasurement last = measurements.get(measurements.size() - 1); assertEquals(0, last.getValue()); - assertAttributes(e.getPartitionContext().getPartitionId(), last.getAttributes()); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, e.getPartitionContext().getPartitionId(), CONSUMER_GROUP, null, + null, last.getAttributes()); }) .expectComplete() .verify(DEFAULT_TIMEOUT); @@ -852,9 +855,9 @@ void receiveNullMeterDoesNotThrow() { @Test void startSpanForGetProperties() { // Arrange - final Tracer tracer1 = mock(Tracer.class); - when(tracer1.isEnabled()).thenReturn(true); - EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer1, null, + final Tracer tracer = mock(Tracer.class); + when(tracer.isEnabled()).thenReturn(true); + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, null, HOSTNAME, EVENT_HUB_NAME, CONSUMER_GROUP, false); EventHubConsumerAsyncClient consumer = new EventHubConsumerAsyncClient(HOSTNAME, EVENT_HUB_NAME, connectionProcessor, messageSerializer, CONSUMER_GROUP, PREFETCH, false, onClientClosed, CLIENT_IDENTIFIER, @@ -868,16 +871,25 @@ void startSpanForGetProperties() { when(managementNode.getEventHubProperties()).thenReturn(Mono.just(ehProperties)); when(managementNode.getPartitionProperties(anyString())).thenReturn(Mono.just(partitionProperties)); - when(tracer1.start(eq("EventHubs.getPartitionProperties"), any(StartSpanOptions.class), any(Context.class))).thenAnswer( + final String expectedPartitionSpanName = getSpanName(GET_PARTITION_PROPERTIES, EVENT_HUB_NAME); + when(tracer.start(eq(expectedPartitionSpanName), any(StartSpanOptions.class), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class)); + StartSpanOptions startOpts = invocation.getArgument(1, StartSpanOptions.class); + assertEquals(SpanKind.CLIENT, startOpts.getSpanKind()); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, PARTITION_ID, CONSUMER_GROUP, null, + GET_PARTITION_PROPERTIES, startOpts.getAttributes()); return invocation.getArgument(2, Context.class) .addData(PARENT_TRACE_CONTEXT_KEY, "getPartitionProperties"); } ); - when(tracer1.start(eq("EventHubs.getEventHubProperties"), any(StartSpanOptions.class), any(Context.class))).thenAnswer( + + final String expectedHubSpanName = getSpanName(GET_EVENT_HUB_PROPERTIES, EVENT_HUB_NAME); + when(tracer.start(eq(expectedHubSpanName), any(StartSpanOptions.class), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class)); + StartSpanOptions startOpts = invocation.getArgument(1, StartSpanOptions.class); + assertEquals(SpanKind.CLIENT, startOpts.getSpanKind()); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, null, CONSUMER_GROUP, null, + GET_EVENT_HUB_PROPERTIES, startOpts.getAttributes()); return invocation.getArgument(2, Context.class) .addData(PARENT_TRACE_CONTEXT_KEY, "getEventHubsProperties"); } @@ -889,17 +901,17 @@ void startSpanForGetProperties() { .expectComplete() .verify(DEFAULT_TIMEOUT); - StepVerifier.create(consumer.getPartitionProperties("0")) + StepVerifier.create(consumer.getPartitionProperties(PARTITION_ID)) .consumeNextWith(p -> assertSame(partitionProperties, p)) .expectComplete() .verify(DEFAULT_TIMEOUT); //Assert - verify(tracer1, times(1)) - .start(eq("EventHubs.getPartitionProperties"), any(StartSpanOptions.class), any(Context.class)); - verify(tracer1, times(1)) - .start(eq("EventHubs.getEventHubProperties"), any(StartSpanOptions.class), any(Context.class)); - verify(tracer1, times(2)).end(isNull(), isNull(), any()); + verify(tracer, times(1)) + .start(eq(expectedPartitionSpanName), any(StartSpanOptions.class), any(Context.class)); + verify(tracer, times(1)) + .start(eq(expectedHubSpanName), any(StartSpanOptions.class), any(Context.class)); + verify(tracer, times(2)).end(isNull(), isNull(), any()); verifyNoInteractions(onClientClosed); } @@ -952,12 +964,6 @@ void getPropertiesWithRetries() { assertEquals(3, tryCount.get()); } - private void assertStartOptions(StartSpanOptions startOpts) { - assertEquals(SpanKind.CLIENT, startOpts.getSpanKind()); - assertEquals(EVENT_HUB_NAME, startOpts.getAttributes().get(ENTITY_PATH_KEY)); - assertEquals(HOSTNAME, startOpts.getAttributes().get(HOST_NAME_KEY)); - } - private void assertPartition(String partitionId, PartitionEvent event) { LOGGER.log(LogLevel.VERBOSE, () -> "Event received: " + event.getPartitionContext().getPartitionId()); final Object value = event.getData().getProperties().get(PARTITION_ID_HEADER); @@ -986,14 +992,6 @@ private void sendMessages(TestPublisher testPublisher, int numberOfEven } } - private void assertAttributes(String entityPath, Map attributes) { - assertEquals(4, attributes.size()); - assertEquals(HOSTNAME, attributes.get("hostName")); - assertEquals(EVENT_HUB_NAME, attributes.get("entityName")); - assertEquals(CONSUMER_GROUP, attributes.get("consumerGroup")); - assertEquals(entityPath, attributes.get("partitionId")); - } - private ConnectionCacheWrapper createConnectionProcessor(EventHubAmqpConnection connection, AmqpRetryOptions retryOptions, boolean isV2) { if (isV2) { final AmqpRetryPolicy retryPolicy = RetryUtil.getRetryPolicy(retryOptions); diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClientTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClientTest.java index 1305a9a6c24b7..5a618f4edb226 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClientTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClientTest.java @@ -20,6 +20,7 @@ import com.azure.core.amqp.models.CbsAuthorizationType; import com.azure.core.credential.TokenCredential; import com.azure.core.test.utils.metrics.TestCounter; +import com.azure.core.test.utils.metrics.TestHistogram; import com.azure.core.test.utils.metrics.TestMeasurement; import com.azure.core.test.utils.metrics.TestMeter; import com.azure.core.util.ClientOptions; @@ -50,11 +51,17 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import reactor.core.Exceptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.Sinks; @@ -65,6 +72,7 @@ import java.time.Duration; import java.time.Instant; import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -76,16 +84,25 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Stream; import static com.azure.core.amqp.AmqpMessageConstant.ENQUEUED_TIME_UTC_ANNOTATION_NAME; import static com.azure.core.amqp.AmqpMessageConstant.OFFSET_ANNOTATION_NAME; import static com.azure.core.amqp.AmqpMessageConstant.SEQUENCE_NUMBER_ANNOTATION_NAME; -import static com.azure.core.util.tracing.Tracer.ENTITY_PATH_KEY; -import static com.azure.core.util.tracing.Tracer.HOST_NAME_KEY; import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; import static com.azure.core.util.tracing.Tracer.SPAN_CONTEXT_KEY; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.DIAGNOSTIC_ID_KEY; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.TRACEPARENT_KEY; +import static com.azure.messaging.eventhubs.TestUtils.assertAllAttributes; +import static com.azure.messaging.eventhubs.TestUtils.assertAttributes; +import static com.azure.messaging.eventhubs.TestUtils.getSpanName; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.DIAGNOSTIC_ID_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_TYPE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.TRACEPARENT_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.getOperationType; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.EVENT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_EVENT_HUB_PROPERTIES; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_PARTITION_PROPERTIES; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.SEND; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -352,16 +369,18 @@ void sendStartSpanSingleMessage() { when(sendLink.send(anyList())).thenReturn(Mono.empty()); when(sendLink.send(any(Message.class))).thenReturn(Mono.empty()); - when(tracer1.start(eq("EventHubs.message"), any(), any(Context.class))).thenAnswer( + final String expectedMessageSpanName = getSpanName(EVENT, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedMessageSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, 0); + assertStartOptions(EVENT, invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, null, 0); return invocation.getArgument(2, Context.class) .addData(SPAN_CONTEXT_KEY, "span"); }); - when(tracer1.start(eq("EventHubs.send"), any(), any(Context.class))).thenAnswer( + final String expectedSendSpanName = getSpanName(SEND, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedSendSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 1); + assertStartOptions(SEND, invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, null, 1); return invocation.getArgument(2, Context.class) .addData(PARENT_TRACE_CONTEXT_KEY, "trace-context"); } @@ -380,9 +399,9 @@ void sendStartSpanSingleMessage() { //Assert verify(tracer1, times(1)) - .start(eq("EventHubs.send"), any(), any(Context.class)); + .start(eq(expectedSendSpanName), any(), any(Context.class)); verify(tracer1, times(1)) - .start(eq("EventHubs.message"), any(), any(Context.class)); + .start(eq(expectedMessageSpanName), any(), any(Context.class)); verify(tracer1, times(2)).end(isNull(), isNull(), any()); verify(tracer1, times(1)).injectContext(any(), any()); @@ -414,16 +433,18 @@ void sendSingleWithUnmodifiableProperties() { when(sendLink.send(anyList())).thenReturn(Mono.empty()); when(sendLink.send(any(Message.class))).thenReturn(Mono.empty()); - when(tracer1.start(eq("EventHubs.message"), any(), any(Context.class))).thenAnswer( + final String expectedMessageSpanName = getSpanName(EVENT, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedMessageSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, 0); + assertStartOptions(EVENT, invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, null, 0); return invocation.getArgument(2, Context.class) .addData(SPAN_CONTEXT_KEY, "span"); }); - when(tracer1.start(eq("EventHubs.send"), any(), any(Context.class))).thenAnswer( + final String expectedSendSpanName = getSpanName(SEND, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedSendSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 1); + assertStartOptions(SEND, invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, null, 1); return invocation.getArgument(2, Context.class) .addData(PARENT_TRACE_CONTEXT_KEY, "trace-context"); } @@ -436,9 +457,9 @@ void sendSingleWithUnmodifiableProperties() { //Assert verify(tracer1, times(1)) - .start(eq("EventHubs.send"), any(), any(Context.class)); + .start(eq(expectedSendSpanName), any(), any(Context.class)); verify(tracer1, times(1)) - .start(eq("EventHubs.message"), any(), any(Context.class)); + .start(eq(expectedMessageSpanName), any(), any(Context.class)); verify(tracer1, times(1)).end(eq("failed to inject context into EventData"), isNull(), any()); verify(tracer1, times(1)).end(isNull(), isNull(), any()); verify(tracer1, never()).injectContext(any(), any()); @@ -460,23 +481,27 @@ void startSpanForGetProperties() { connectionProcessor, retryOptions, messageSerializer, Schedulers.parallel(), false, onClientClosed, CLIENT_IDENTIFIER, instrumentation); - EventHubProperties ehProperties = new EventHubProperties(EVENT_HUB_NAME, Instant.now(), new String[]{"0"}); - PartitionProperties partitionProperties = new PartitionProperties(EVENT_HUB_NAME, "0", + final String partitionId = "0"; + EventHubProperties ehProperties = new EventHubProperties(EVENT_HUB_NAME, Instant.now(), new String[]{partitionId}); + PartitionProperties partitionProperties = new PartitionProperties(EVENT_HUB_NAME, partitionId, 1L, 2L, OffsetDateTime.now().toString(), Instant.now(), false); EventHubManagementNode managementNode = mock(EventHubManagementNode.class); when(connection.getManagementNode()).thenReturn(Mono.just(managementNode)); when(managementNode.getEventHubProperties()).thenReturn(Mono.just(ehProperties)); when(managementNode.getPartitionProperties(anyString())).thenReturn(Mono.just(partitionProperties)); - when(tracer1.start(eq("EventHubs.getPartitionProperties"), any(), any(Context.class))).thenAnswer( + final String expectedPartitionSpanName = getSpanName(GET_PARTITION_PROPERTIES, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedPartitionSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 0); + assertStartOptions(GET_PARTITION_PROPERTIES, invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, partitionId, 0); return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "getPartitionProperties"); } ); - when(tracer1.start(eq("EventHubs.getEventHubProperties"), any(), any(Context.class))).thenAnswer( + + final String expectedHubPropertiesSpanName = getSpanName(GET_EVENT_HUB_PROPERTIES, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedHubPropertiesSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 0); + assertStartOptions(GET_EVENT_HUB_PROPERTIES, invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, null, 0); return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "getEventHubProperties"); } ); @@ -494,9 +519,9 @@ void startSpanForGetProperties() { //Assert verify(tracer1, times(1)) - .start(eq("EventHubs.getPartitionProperties"), any(), any(Context.class)); + .start(eq(expectedPartitionSpanName), any(), any(Context.class)); verify(tracer1, times(1)) - .start(eq("EventHubs.getEventHubProperties"), any(), any(Context.class)); + .start(eq(expectedHubPropertiesSpanName), any(), any(Context.class)); verify(tracer1, times(2)).end(isNull(), isNull(), any()); verifyNoInteractions(onClientClosed); @@ -574,9 +599,10 @@ void sendMessageRetrySpanTest() { .addContext(SPAN_CONTEXT_KEY, "span-context"); testData.getProperties().put(failureKey, "true"); - when(tracer1.start(eq("EventHubs.send"), any(), any(Context.class))).thenAnswer( + final String expectedSendSpanName = getSpanName(SEND, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedSendSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 1); + assertStartOptions(SEND, invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, null, 1); return invocation.getArgument(2, Context.class) .addData(PARENT_TRACE_CONTEXT_KEY, "trace-context"); } @@ -608,8 +634,8 @@ void sendMessageRetrySpanTest() { assertFalse(testData.getProperties().containsKey(DIAGNOSTIC_ID_KEY)); //Assert - verify(tracer1, times(1)).start(eq("EventHubs.send"), any(), any(Context.class)); - verify(tracer1, never()).start(eq("EventHubs.message"), any(), any(Context.class)); + verify(tracer1, times(1)).start(eq(expectedSendSpanName), any(), any(Context.class)); + verify(tracer1, never()).start(eq(getSpanName(EVENT, EVENT_HUB_NAME)), any(), any(Context.class)); verify(tracer1, times(1)).end(isNull(), isNull(), any()); verify(tracer1, never()).extractContext(any()); verify(tracer1, never()).injectContext(any(), any()); @@ -695,9 +721,11 @@ void createsEventDataBatch() { * Verifies that message spans are started and ended on tryAdd when creating batches to send in {@link * EventDataBatch}. */ - @Test + @ParameterizedTest + @NullSource + @ValueSource(strings = {"0"}) @SuppressWarnings("unchecked") - void startMessageSpansOnCreateBatch() { + void startMessageSpansOnCreateBatch(String partitionId) { // Arrange final Tracer tracer1 = mock(Tracer.class); when(tracer1.isEnabled()).thenReturn(true); @@ -706,31 +734,32 @@ void startMessageSpansOnCreateBatch() { connectionProcessor, retryOptions, messageSerializer, Schedulers.parallel(), false, onClientClosed, CLIENT_IDENTIFIER, instrumentation); final AmqpSendLink link = mock(AmqpSendLink.class); + final String entityPath = partitionId == null ? EVENT_HUB_NAME : String.format("%s/Partitions/%s", EVENT_HUB_NAME, partitionId); when(link.getLinkSize()).thenReturn(Mono.just(ClientConstants.MAX_MESSAGE_LENGTH_BYTES)); when(link.getHostname()).thenReturn(HOSTNAME); - when(link.getEntityPath()).thenReturn(ENTITY_PATH); + when(link.getEntityPath()).thenReturn(entityPath); - // EC is the prefix they use when creating a link that sends to the service round-robin. - when(connection.createSendLink(eq(EVENT_HUB_NAME), eq(EVENT_HUB_NAME), eq(retryOptions), eq(CLIENT_IDENTIFIER))) + when(connection.createSendLink(eq(entityPath), eq(entityPath), eq(retryOptions), eq(CLIENT_IDENTIFIER))) .thenReturn(Mono.just(sendLink)); when(sendLink.getHostname()).thenReturn(HOSTNAME); - when(sendLink.getEntityPath()).thenReturn(EVENT_HUB_NAME); + when(sendLink.getEntityPath()).thenReturn(entityPath); when(sendLink.send(anyList())).thenReturn(Mono.empty()); when(sendLink.send(any(Message.class))).thenReturn(Mono.empty()); final AtomicReference eventInd = new AtomicReference<>(0); - - when(tracer1.start(eq("EventHubs.message"), any(), any(Context.class))).thenAnswer( + final String expectedMessageSpanName = getSpanName(EVENT, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedMessageSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, 0); + assertStartOptions(EVENT, invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, null, 0); return invocation.getArgument(2, Context.class) .addData(SPAN_CONTEXT_KEY, "span"); }); - when(tracer1.start(eq("EventHubs.send"), any(), any(Context.class))).thenAnswer( + final String expectedSendSpanName = getSpanName(SEND, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedSendSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 2); + assertStartOptions(SEND, invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, partitionId, 2); return invocation.getArgument(2, Context.class) .addData(PARENT_TRACE_CONTEXT_KEY, "trace-context"); } @@ -743,7 +772,7 @@ void startMessageSpansOnCreateBatch() { }).when(tracer1).injectContext(any(), any(Context.class)); // Act & Assert - StepVerifier.create(asyncProducer.createBatch() + StepVerifier.create(asyncProducer.createBatch(new CreateBatchOptions().setPartitionId(partitionId)) .flatMap(batch -> { final EventData data0 = new EventData("Hello World".getBytes(UTF_8)); Assertions.assertTrue(batch.tryAdd(data0)); @@ -759,9 +788,9 @@ void startMessageSpansOnCreateBatch() { .verify(DEFAULT_TIMEOUT); verify(tracer1, times(2)) - .start(eq("EventHubs.message"), any(), any(Context.class)); - verify(tracer1, times(1)).start(eq("EventHubs.send"), any(), any(Context.class)); - verify(tracer1, times(2)).start(eq("EventHubs.message"), any(), any(Context.class)); + .start(eq(expectedMessageSpanName), any(), any(Context.class)); + verify(tracer1, times(1)).start(eq(expectedSendSpanName), any(), any(Context.class)); + verify(tracer1, times(2)).start(eq(expectedMessageSpanName), any(), any(Context.class)); verify(tracer1, times(3)).end(isNull(), isNull(), any()); verify(tracer1, times(2)).injectContext(any(), any()); @@ -1026,7 +1055,7 @@ void sendsAnEventDataBatchWithMetrics() { .expectComplete() .verify(DEFAULT_TIMEOUT); - TestCounter eventCounter = meter.getCounters().get("messaging.eventhubs.events.sent"); + TestCounter eventCounter = meter.getCounters().get("messaging.client.published.messages"); assertNotNull(eventCounter); List> measurements = eventCounter.getMeasurements(); @@ -1034,18 +1063,29 @@ void sendsAnEventDataBatchWithMetrics() { assertEquals(1, measurements.get(0).getValue()); assertEquals(2, measurements.get(1).getValue()); - assertAttributes(eventHub1, null, null, measurements.get(0).getAttributes()); - assertAttributes(eventHub2, null, null, measurements.get(1).getAttributes()); + assertAttributes(HOSTNAME, eventHub1, SEND, measurements.get(0).getAttributes()); + assertAttributes(HOSTNAME, eventHub2, SEND, measurements.get(1).getAttributes()); } + public static Stream errorSource() { + Throwable inner = new RuntimeException("test"); + final ArrayList arguments = new ArrayList<>(); + arguments.add(Arguments.of(inner, inner.getClass().getName())); + arguments.add(Arguments.of(Exceptions.propagate(inner), inner.getClass().getName())); + arguments.add( + Arguments.of( + new AmqpException(false, AmqpErrorCondition.SERVER_BUSY_ERROR, "test", inner, null), + AmqpErrorCondition.SERVER_BUSY_ERROR.getErrorCondition())); + return arguments.stream(); + } - @Test - void sendsAnEventDataBatchWithMetricsFailure() { + @ParameterizedTest + @MethodSource("errorSource") + void sendsAnEventDataBatchWithMetricsFailure(Throwable error, String expectedErrorType) { when(connection.createSendLink(eq(EVENT_HUB_NAME), eq(EVENT_HUB_NAME), any(), any())).thenReturn(Mono.just(sendLink)); - when(sendLink.send(anyList())).thenReturn(Mono.empty()); when(sendLink.getHostname()).thenReturn(HOSTNAME); when(sendLink.getEntityPath()).thenReturn(EVENT_HUB_NAME); - when(sendLink.send(any(Message.class))).thenReturn(Mono.error(new RuntimeException("foo"))); + when(sendLink.send(any(Message.class))).thenReturn(Mono.error(error)); TestMeter meter = new TestMeter(); EventHubsProducerInstrumentation instrumentation = new EventHubsProducerInstrumentation(null, meter, HOSTNAME, EVENT_HUB_NAME); @@ -1055,17 +1095,11 @@ void sendsAnEventDataBatchWithMetricsFailure() { messageSerializer, testScheduler, false, onClientClosed, CLIENT_IDENTIFIER, instrumentation); StepVerifier.create(producer.send(new EventData("1"))) - .expectErrorMessage("foo") + .expectErrorMessage(error.getMessage()) .verify(DEFAULT_TIMEOUT); - TestCounter eventCounter = meter.getCounters().get("messaging.eventhubs.events.sent"); - assertNotNull(eventCounter); - - List> measurements = eventCounter.getMeasurements(); - assertEquals(1, measurements.size()); - - assertEquals(1, measurements.get(0).getValue()); - assertAttributes(EVENT_HUB_NAME, null, "error", measurements.get(0).getAttributes()); + assertSendCount(meter, null, 1, expectedErrorType, null); + assertSendDuration(meter, null, null, expectedErrorType, null); } @Test @@ -1089,14 +1123,8 @@ void sendsAnEventDataBatchWithMetricsPartitionId() { .expectComplete() .verify(DEFAULT_TIMEOUT); - TestCounter eventCounter = meter.getCounters().get("messaging.eventhubs.events.sent"); - assertNotNull(eventCounter); - - List> measurements = eventCounter.getMeasurements(); - assertEquals(1, measurements.size()); - - assertEquals(1, measurements.get(0).getValue()); - assertAttributes(EVENT_HUB_NAME, partitionId, null, measurements.get(0).getAttributes()); + assertSendCount(meter, partitionId, 1, null, null); + assertSendDuration(meter, partitionId, null, null, null); } @Test @@ -1118,16 +1146,18 @@ void sendsAnEventDataBatchWithMetricsAndTraces() { final AtomicReference eventInd = new AtomicReference<>(0); - when(tracer.start(eq("EventHubs.message"), any(), any(Context.class))).thenAnswer( + final String expectedMessageSpanName = getSpanName(EVENT, EVENT_HUB_NAME); + when(tracer.start(eq(expectedMessageSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, 0); + assertStartOptions(EVENT, invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, null, 0); return invocation.getArgument(2, Context.class) .addData(SPAN_CONTEXT_KEY, "span"); }); - when(tracer.start(eq("EventHubs.send"), any(), any(Context.class))).thenAnswer( + final String expectedSendSpanName = getSpanName(SEND, EVENT_HUB_NAME); + when(tracer.start(eq(expectedSendSpanName), any(), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 1); + assertStartOptions(SEND, invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, null, 1); return invocation.getArgument(2, Context.class) .addData(PARENT_TRACE_CONTEXT_KEY, "parent span"); } @@ -1143,16 +1173,8 @@ void sendsAnEventDataBatchWithMetricsAndTraces() { .expectComplete() .verify(DEFAULT_TIMEOUT); - TestCounter eventCounter = meter.getCounters().get("messaging.eventhubs.events.sent"); - assertNotNull(eventCounter); - - List> measurements = eventCounter.getMeasurements(); - assertEquals(1, measurements.size()); - - assertEquals(1, measurements.get(0).getValue()); - assertAttributes(EVENT_HUB_NAME, null, null, measurements.get(0).getAttributes()); - - assertEquals("parent span", measurements.get(0).getContext().getData(PARENT_TRACE_CONTEXT_KEY).get()); + assertSendCount(meter, null, 1, null, "parent span"); + assertSendDuration(meter, null, null, null, "parent span"); } @Test @@ -1173,7 +1195,8 @@ void sendsAnEventDataBatchWithDisabledMetrics() { .expectComplete() .verify(DEFAULT_TIMEOUT); - assertFalse(meter.getCounters().containsKey("messaging.eventhubs.events.sent")); + assertFalse(meter.getCounters().containsKey("messaging.client.published.messages")); + assertFalse(meter.getHistograms().containsKey("messaging.client.operation.duration")); } @Test @@ -1512,22 +1535,6 @@ void resendMessageOnTransientLinkFailure() { verifyNoInteractions(onClientClosed); } - private void assertAttributes(String entityName, String entityPath, String status, Map attributes) { - int expectedAttributeCount = 4; - if (entityPath == null) { - expectedAttributeCount--; - } - if (status == null) { - expectedAttributeCount--; - } - - assertEquals(expectedAttributeCount, attributes.size()); - assertEquals(HOSTNAME, attributes.get("hostName")); - assertEquals(entityName, attributes.get("entityName")); - assertEquals(entityPath, attributes.get("partitionId")); - assertEquals(status, attributes.get("status")); - } - private EventData fakeReceivedMessage() { Message receivedMessage = Proton.message(); receivedMessage.setApplicationProperties(new ApplicationProperties(Collections.singletonMap("foo", "bar"))); @@ -1543,10 +1550,13 @@ private EventData fakeReceivedMessage() { return serializer.deserialize(receivedMessage, EventData.class); } - private void assertStartOptions(StartSpanOptions startOpts, SpanKind kind, int linkCount) { + private void assertStartOptions( + com.azure.messaging.eventhubs.implementation.instrumentation.OperationName operationName, StartSpanOptions startOpts, SpanKind kind, String partitionId, int linkCount) { assertEquals(kind, startOpts.getSpanKind()); - assertEquals(EVENT_HUB_NAME, startOpts.getAttributes().get(ENTITY_PATH_KEY)); - assertEquals(HOSTNAME, startOpts.getAttributes().get(HOST_NAME_KEY)); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, partitionId, null, null, operationName, + startOpts.getAttributes()); + assertEquals(operationName.toString(), startOpts.getAttributes().get(MESSAGING_OPERATION_NAME)); + assertEquals(getOperationType(operationName), startOpts.getAttributes().get(MESSAGING_OPERATION_TYPE)); if (linkCount == 0) { assertNull(startOpts.getLinks()); @@ -1601,6 +1611,36 @@ private ConnectionCacheWrapper createConnectionProcessor(EventHubReactorAmqpConn } } + private void assertSendCount(TestMeter meter, String partitionId, int expectedValue, String expectedErrorType, Object parentContext) { + TestCounter eventCounter = meter.getCounters().get("messaging.client.published.messages"); + assertNotNull(eventCounter); + + List> measurements = eventCounter.getMeasurements(); + assertEquals(1, measurements.size()); + assertEquals(expectedValue, measurements.get(0).getValue(), expectedValue); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, partitionId, null, expectedErrorType, + SEND, measurements.get(0).getAttributes()); + if (parentContext != null) { + assertEquals(parentContext, measurements.get(0).getContext().getData(PARENT_TRACE_CONTEXT_KEY).get()); + } + } + + private void assertSendDuration(TestMeter meter, String partitionId, Double expectedValue, String expectedErrorType, Object parentContext) { + TestHistogram sendDuration = meter.getHistograms().get("messaging.client.operation.duration"); + assertNotNull(sendDuration); + List> measurements = sendDuration.getMeasurements(); + assertEquals(1, measurements.size()); + if (expectedValue != null) { + assertEquals(expectedValue, measurements.get(0).getValue(), expectedValue); + } + + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, partitionId, null, expectedErrorType, + SEND, measurements.get(0).getAttributes()); + if (parentContext != null) { + assertEquals(parentContext, measurements.get(0).getContext().getData(PARENT_TRACE_CONTEXT_KEY).get()); + } + } + private static final String TEST_CONTENTS = "SSLorem ipsum dolor sit amet, consectetur adipiscing elit. Donec " + "vehicula posuere lobortis. Aliquam finibus volutpat dolor, faucibus pellentesque ipsum bibendum vitae. " + "Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Ut sit amet " diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerClientTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerClientTest.java index 00b35b4ffad43..947243768d4bd 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerClientTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerClientTest.java @@ -48,20 +48,28 @@ import java.time.Duration; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Function; -import static com.azure.core.util.tracing.Tracer.ENTITY_PATH_KEY; -import static com.azure.core.util.tracing.Tracer.HOST_NAME_KEY; import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; import static com.azure.core.util.tracing.Tracer.SPAN_CONTEXT_KEY; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.DIAGNOSTIC_ID_KEY; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.TRACEPARENT_KEY; +import static com.azure.messaging.eventhubs.TestUtils.getSpanName; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.DIAGNOSTIC_ID_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_SYSTEM; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_SYSTEM_VALUE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.SERVER_ADDRESS; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.TRACEPARENT_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.EVENT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.SEND; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; @@ -185,14 +193,16 @@ public void sendStartSpanSingleMessage() { when(connection.createSendLink(eq(EVENT_HUB_NAME), eq(EVENT_HUB_NAME), any(), eq(CLIENT_IDENTIFIER))) .thenReturn(Mono.just(sendLink)); - when(tracer1.start(eq("EventHubs.message"), any(), any(Context.class))).thenAnswer( + final String expectedMessageSpanName = getSpanName(EVENT, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedMessageSpanName), any(), any(Context.class))).thenAnswer( invocation -> { assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, 0); return invocation.getArgument(2, Context.class) .addData(SPAN_CONTEXT_KEY, "span"); }); - when(tracer1.start(eq("EventHubs.send"), any(), any(Context.class))).thenAnswer( + final String expectedSendSpanName = getSpanName(SEND, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedSendSpanName), any(), any(Context.class))).thenAnswer( invocation -> { assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 1); return invocation.getArgument(2, Context.class) @@ -217,9 +227,9 @@ public void sendStartSpanSingleMessage() { //Assert verify(tracer1, times(1)) - .start(eq("EventHubs.send"), any(), any(Context.class)); + .start(eq(expectedSendSpanName), any(), any(Context.class)); verify(tracer1, times(1)) - .start(eq("EventHubs.message"), any(), any(Context.class)); + .start(eq(expectedMessageSpanName), any(), any(Context.class)); verify(tracer1, times(2)).end(isNull(), isNull(), any()); verify(tracer1, times(1)).injectContext(any(), any()); @@ -248,7 +258,9 @@ public void sendMessageRetrySpanTest() { final EventData eventData = new EventData("hello-world".getBytes(UTF_8)); eventData.getProperties().put("traceparent", "traceparent"); - when(tracer1.start(eq("EventHubs.send"), any(), any(Context.class))).thenAnswer( + final String expectedSendSpanName = getSpanName(SEND, EVENT_HUB_NAME); + final String expectedMessageSpanName = getSpanName(EVENT, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedSendSpanName), any(), any(Context.class))).thenAnswer( invocation -> { assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 1); return invocation.getArgument(2, Context.class) @@ -273,10 +285,10 @@ public void sendMessageRetrySpanTest() { } //Assert - verify(tracer1, times(1)).start(eq("EventHubs.send"), any(), any(Context.class)); + verify(tracer1, times(1)).start(eq(expectedSendSpanName), any(), any(Context.class)); verify(tracer1, times(1)).end(isNull(), isNull(), any()); verify(tracer1, times(1)).extractContext(any()); - verify(tracer1, never()).start(eq("EventHubs.message"), any(), any(Context.class)); + verify(tracer1, never()).start(eq(expectedMessageSpanName), any(), any(Context.class)); verify(tracer1, never()).injectContext(any(), any()); verifyNoInteractions(onClientClosed); } @@ -408,15 +420,16 @@ public void startsMessageSpanOnEventBatch() { .thenReturn(Mono.just(sendLink)); final AtomicReference eventInd = new AtomicReference<>(0); - - when(tracer1.start(eq("EventHubs.message"), any(), any(Context.class))).thenAnswer( + final String expectedMessageSpanName = getSpanName(EVENT, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedMessageSpanName), any(), any(Context.class))).thenAnswer( invocation -> { assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.PRODUCER, 0); return invocation.getArgument(2, Context.class) .addData(SPAN_CONTEXT_KEY, "span"); }); - when(tracer1.start(eq("EventHubs.send"), any(), any(Context.class))).thenAnswer( + final String expectedSendSpanName = getSpanName(SEND, EVENT_HUB_NAME); + when(tracer1.start(eq(expectedSendSpanName), any(), any(Context.class))).thenAnswer( invocation -> { assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), SpanKind.CLIENT, 2); return invocation.getArgument(2, Context.class) @@ -447,9 +460,9 @@ public void startsMessageSpanOnEventBatch() { } verify(tracer1, times(2)) - .start(eq("EventHubs.message"), any(), any(Context.class)); - verify(tracer1, times(1)).start(eq("EventHubs.send"), any(), any(Context.class)); - verify(tracer1, times(2)).start(eq("EventHubs.message"), any(), any(Context.class)); + .start(eq(expectedMessageSpanName), any(), any(Context.class)); + verify(tracer1, times(1)).start(eq(expectedSendSpanName), any(), any(Context.class)); + verify(tracer1, times(2)).start(eq(expectedMessageSpanName), any(), any(Context.class)); verify(tracer1, times(3)).end(isNull(), isNull(), any()); verify(tracer1, times(2)).injectContext(any(), any()); @@ -591,8 +604,12 @@ public void createBatchWithRetry() { private void assertStartOptions(StartSpanOptions startOpts, SpanKind kind, int linkCount) { assertEquals(kind, startOpts.getSpanKind()); - assertEquals(EVENT_HUB_NAME, startOpts.getAttributes().get(ENTITY_PATH_KEY)); - assertEquals(HOSTNAME, startOpts.getAttributes().get(HOST_NAME_KEY)); + + Map attributes = startOpts.getAttributes(); + assertEquals(EVENT_HUB_NAME, attributes.get(MESSAGING_DESTINATION_NAME)); + assertEquals(HOSTNAME, attributes.get(SERVER_ADDRESS)); + assertEquals(MESSAGING_SYSTEM_VALUE, attributes.get(MESSAGING_SYSTEM)); + assertNotNull(attributes.get(MESSAGING_OPERATION_NAME)); if (linkCount == 0) { assertNull(startOpts.getLinks()); diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentationTests.java new file mode 100644 index 0000000000000..4291301f735a1 --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentationTests.java @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs; + +import com.azure.core.test.utils.metrics.TestCounter; +import com.azure.core.test.utils.metrics.TestHistogram; +import com.azure.core.test.utils.metrics.TestMeter; +import com.azure.core.tracing.opentelemetry.OpenTelemetryTracingOptions; +import com.azure.core.util.TracingOptions; +import com.azure.core.util.tracing.Tracer; +import com.azure.core.util.tracing.TracerProvider; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.data.SpanData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.Exceptions; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.azure.messaging.eventhubs.TestUtils.assertAllAttributes; +import static com.azure.messaging.eventhubs.TestUtils.assertSpanStatus; +import static com.azure.messaging.eventhubs.TestUtils.attributesToMap; +import static com.azure.messaging.eventhubs.TestUtils.getSpanName; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.EVENT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.SEND; +import static io.opentelemetry.api.trace.SpanKind.CLIENT; +import static io.opentelemetry.api.trace.SpanKind.PRODUCER; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +public class EventHubsProducerInstrumentationTests { + private static final String FQDN = "fqdn"; + private static final String ENTITY_NAME = "entityName"; + private Tracer tracer; + private TestMeter meter; + private TestSpanProcessor spanProcessor; + + @BeforeEach + public void setup(TestInfo testInfo) { + spanProcessor = new TestSpanProcessor(FQDN, ENTITY_NAME, testInfo.getDisplayName()); + OpenTelemetry otel = OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .addSpanProcessor(spanProcessor) + .build()) + .build(); + + TracingOptions tracingOptions = new OpenTelemetryTracingOptions().setOpenTelemetry(otel); + tracer = TracerProvider.getDefaultProvider() + .createTracer("test", null, "Microsoft.EventHub", tracingOptions); + meter = new TestMeter(); + } + + @AfterEach + public void teardown() { + spanProcessor.shutdown(); + spanProcessor.close(); + meter.close(); + } + + @Test + @SuppressWarnings("try") + public void sendBatchDisabledInstrumentation() { + EventHubsProducerInstrumentation instrumentation = new EventHubsProducerInstrumentation(null, null, + FQDN, ENTITY_NAME); + + EventDataBatch batch = new EventDataBatch(1024, null, null, + () -> null, instrumentation); + batch.tryAdd(new EventData("test")); + Mono inner = Mono.empty(); + assertSame(inner, instrumentation.sendBatch(inner, batch)); + } + + public static Stream sendBatchErrors() { + return Stream.of( + Arguments.of(false, null, null, null), + Arguments.of(true, null, "cancelled", "cancelled"), + Arguments.of(false, new RuntimeException("test"), RuntimeException.class.getName(), "test"), + Arguments.of(false, Exceptions.propagate(new RuntimeException("test")), RuntimeException.class.getName(), "test") + ); + } + + @ParameterizedTest + @MethodSource("sendBatchErrors") + @SuppressWarnings("try") + public void sendBatchOneEvent(boolean cancel, Throwable error, String expectedErrorType, String spanDescription) { + EventHubsProducerInstrumentation instrumentation = new EventHubsProducerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME); + EventDataBatch batch = new EventDataBatch(1024, null, null, + () -> null, instrumentation); + batch.tryAdd(new EventData("test")); + Mono inner = Mono.defer(() -> error == null ? Mono.empty() : Mono.error(error)); + + StepVerifier.FirstStep stepVerifier = + StepVerifier.create(instrumentation.sendBatch(inner, batch)); + + if (cancel) { + stepVerifier.thenCancel().verify(); + } else if (error != null) { + stepVerifier.expectErrorMessage(error.getMessage()).verify(); + } else { + stepVerifier.expectComplete().verify(); + } + + assertSendDuration(null, expectedErrorType); + assertBatchCount(1, null, expectedErrorType); + assertSpans(1, null, expectedErrorType, spanDescription); + } + + @ParameterizedTest + @MethodSource("sendBatchErrors") + @SuppressWarnings("try") + public void sendBatchMultipleEvents(boolean cancel, Throwable error, String expectedErrorType, String spanDescription) { + EventHubsProducerInstrumentation instrumentation = new EventHubsProducerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME); + + String partitionId = "1"; + EventDataBatch batch = new EventDataBatch(1024, partitionId, null, + () -> null, instrumentation); + + int count = 3; + for (int i = 0; i < count; i++) { + batch.tryAdd(new EventData("test" + i)); + } + + Mono inner = Mono.defer(() -> error == null ? Mono.empty() : Mono.error(error)); + + StepVerifier.FirstStep stepVerifier = + StepVerifier.create(instrumentation.sendBatch(inner, batch)); + + if (cancel) { + stepVerifier.thenCancel().verify(); + } else if (error != null) { + stepVerifier.expectErrorMessage(error.getMessage()).verify(); + } else { + stepVerifier.expectComplete().verify(); + } + + assertSendDuration(partitionId, expectedErrorType); + assertBatchCount(count, partitionId, expectedErrorType); + assertSpans(count, partitionId, expectedErrorType, spanDescription); + } + + private void assertSendDuration(String partitionId, String expectedErrorType) { + TestHistogram publishDuration = meter.getHistograms().get("messaging.client.operation.duration"); + assertNotNull(publishDuration); + assertEquals(1, publishDuration.getMeasurements().size()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, null, expectedErrorType, + SEND, publishDuration.getMeasurements().get(0).getAttributes()); + } + + private void assertSpans(int batchSize, String partitionId, String expectedErrorType, String spanDescription) { + assertEquals(batchSize + 1, spanProcessor.getEndedSpans().size()); + List eventSpans = spanProcessor.getEndedSpans().stream() + .filter(s -> s.getKind().equals(PRODUCER)) + .map(s -> s.toSpanData()) + .collect(Collectors.toList()); + assertEquals(batchSize, eventSpans.size()); + + List publishSpans = spanProcessor.getEndedSpans().stream() + .filter(s -> s.getKind().equals(CLIENT)) + .map(s -> s.toSpanData()) + .collect(Collectors.toList()); + assertEquals(1, publishSpans.size()); + + assertEquals(getSpanName(SEND, ENTITY_NAME), publishSpans.get(0).getName()); + Map publishAttributes = attributesToMap(publishSpans.get(0).getAttributes()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, null, expectedErrorType, + SEND, publishAttributes); + assertEquals(Long.valueOf(batchSize), publishAttributes.get("messaging.batch.message_count")); + assertSpanStatus(spanDescription, publishSpans.get(0)); + + List links = publishSpans.get(0).getLinks(); + for (int i = 0; i < batchSize; i++) { + assertEquals(getSpanName(EVENT, ENTITY_NAME), eventSpans.get(i).getName()); + Map eventAttributes = attributesToMap(eventSpans.get(i).getAttributes()); + assertAllAttributes(FQDN, ENTITY_NAME, null, null, null, + EVENT, eventAttributes); + assertSpanStatus(null, eventSpans.get(i)); + + SpanContext createContext = eventSpans.get(i).getSpanContext(); + links.stream() + .filter(l -> spanContextEquals(l.getSpanContext(), createContext)) + .findFirst().orElseThrow(() -> new AssertionError("Link not found")); + } + } + + private static boolean spanContextEquals(SpanContext c1, SpanContext c2) { + // we don't take remote into account, so can't use equals implementation on SpanContext + return c1.getTraceId().equals(c2.getTraceId()) + && c1.getSpanId().equals(c2.getSpanId()) + && c1.getTraceFlags().equals(c2.getTraceFlags()) + && c1.getTraceState().equals(c2.getTraceState()); + } + + private void assertBatchCount(int count, String partitionId, String expectedErrorType) { + TestCounter batchCounter = meter.getCounters().get("messaging.client.published.messages"); + assertNotNull(batchCounter); + assertEquals(1, batchCounter.getMeasurements().size()); + assertEquals(Long.valueOf(count), batchCounter.getMeasurements().get(0).getValue()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, null, expectedErrorType, + SEND, batchCounter.getMeasurements().get(0).getAttributes()); + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientErrorHandlingTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientErrorHandlingTest.java index 4d10696e810b3..7e755bf0c6e14 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientErrorHandlingTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientErrorHandlingTest.java @@ -103,7 +103,7 @@ public void testCheckpointStoreErrors(CheckpointStore checkpointStore) throws In Assertions.assertEquals("NONE", errorContext.getPartitionContext().getPartitionId()); Assertions.assertEquals("cg", errorContext.getPartitionContext().getConsumerGroup()); Assertions.assertTrue(errorContext.getThrowable() instanceof IllegalStateException); - }, tracer, processorOptions); + }, tracer, null, processorOptions); client.start(); boolean completed = countDownLatch.await(3, TimeUnit.SECONDS); try { @@ -134,7 +134,7 @@ public void testProcessEventHandlerError() throws InterruptedException { EventProcessorClient client = new EventProcessorClient(eventHubClientBuilder, () -> new BadProcessEventHandler(countDownLatch), new SampleCheckpointStore(), - errorContext -> { }, tracer, processorOptions); + errorContext -> { }, tracer, null, processorOptions); client.start(); boolean completed = countDownLatch.await(3, TimeUnit.SECONDS); client.stop(); @@ -160,7 +160,7 @@ public void testInitHandlerError() throws InterruptedException { EventProcessorClient client = new EventProcessorClient(eventHubClientBuilder, () -> new BadInitHandler(countDownLatch), new SampleCheckpointStore(), - errorContext -> { }, tracer, processorOptions); + errorContext -> { }, tracer, null, processorOptions); client.start(); boolean completed = countDownLatch.await(3, TimeUnit.SECONDS); client.stop(); @@ -187,7 +187,7 @@ public void testCloseHandlerError() throws InterruptedException { EventProcessorClient client = new EventProcessorClient(eventHubClientBuilder, () -> new BadCloseHandler(countDownLatch), new SampleCheckpointStore(), - errorContext -> { }, tracer, processorOptions); + errorContext -> { }, tracer, null, processorOptions); client.start(); boolean completed = countDownLatch.await(3, TimeUnit.SECONDS); diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientTest.java index e3b228d0989e0..18769507e6757 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientTest.java @@ -3,12 +3,16 @@ package com.azure.messaging.eventhubs; +import com.azure.core.test.utils.metrics.TestCounter; +import com.azure.core.test.utils.metrics.TestHistogram; +import com.azure.core.test.utils.metrics.TestMeter; import com.azure.core.util.Context; import com.azure.core.util.tracing.SpanKind; import com.azure.core.util.tracing.StartSpanOptions; import com.azure.core.util.tracing.Tracer; import com.azure.core.util.tracing.TracingLink; import com.azure.messaging.eventhubs.implementation.PartitionProcessor; +import com.azure.messaging.eventhubs.implementation.instrumentation.*; import com.azure.messaging.eventhubs.models.ErrorContext; import com.azure.messaging.eventhubs.models.EventBatchContext; import com.azure.messaging.eventhubs.models.EventContext; @@ -20,9 +24,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import reactor.core.Exceptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; @@ -41,15 +49,19 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import java.util.stream.Stream; -import static com.azure.core.util.tracing.Tracer.ENTITY_PATH_KEY; -import static com.azure.core.util.tracing.Tracer.HOST_NAME_KEY; import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; import static com.azure.core.util.tracing.Tracer.SPAN_CONTEXT_KEY; import static com.azure.messaging.eventhubs.EventHubClientBuilder.DEFAULT_PREFETCH_COUNT; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.DIAGNOSTIC_ID_KEY; -import static com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer.MESSAGE_ENQUEUED_TIME_ATTRIBUTE_NAME; +import static com.azure.messaging.eventhubs.TestUtils.assertAllAttributes; +import static com.azure.messaging.eventhubs.TestUtils.getSpanName; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.DIAGNOSTIC_ID_KEY; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.PROCESS; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.CHECKPOINT; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertIterableEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -61,10 +73,12 @@ import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -73,6 +87,10 @@ * Unit tests for {@link EventProcessorClient}. */ public class EventProcessorClientTest { + private static final String HOSTNAME = "test-ns"; + private static final String EVENT_HUB_NAME = "test-eh"; + private static final String CONSUMER_GROUP = "test-consumer"; + private static final String PARTITION_ID = "1"; private AutoCloseable mocksDisposable; @Mock @@ -121,17 +139,12 @@ public void teardown() throws Exception { */ @Test public void testWithSimplePartitionProcessor() throws Exception { - Tracer tracer = mock(Tracer.class); // Arrange - final String consumerGroup = "test-consumer"; - final String eventHubName = "test-event-hub"; - final String fullyQualifiedNamespace = "test-namespace"; - when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); - when(eventHubClientBuilder.createTracer()).thenReturn(tracer); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(fullyQualifiedNamespace); - when(eventHubAsyncClient.getEventHubName()).thenReturn(eventHubName); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubClientBuilder.createTracer()).thenReturn(null); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); when(eventHubAsyncClient .createConsumer(anyString(), anyInt(), anyBoolean())) @@ -148,16 +161,8 @@ public void testWithSimplePartitionProcessor() throws Exception { final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(); final long beforeTest = System.currentTimeMillis(); - String diagnosticId = "00-08ee063508037b1719dddcbf248e30e2-1365c684eb25daed-01"; - when(tracer.extractContext(any())).thenAnswer(invocation -> new Context(SPAN_CONTEXT_KEY, "value")); - when(tracer.start(eq("EventHubs.process"), any(), any(Context.class))).thenAnswer( - invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), 0); - return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "value2"); - } - ); - processorOptions.setConsumerGroup(consumerGroup) + processorOptions.setConsumerGroup(CONSUMER_GROUP) .setTrackLastEnqueuedEventProperties(false) .setInitialEventPositionProvider(null) .setMaxBatchSize(1) @@ -169,7 +174,7 @@ public void testWithSimplePartitionProcessor() throws Exception { // Act final EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, tracer, + () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, null, processorOptions); eventProcessorClient.start(); @@ -178,14 +183,14 @@ public void testWithSimplePartitionProcessor() throws Exception { // Assert assertNotNull(eventProcessorClient.getIdentifier()); - StepVerifier.create(checkpointStore.listOwnership(fullyQualifiedNamespace, eventHubName, consumerGroup)) + StepVerifier.create(checkpointStore.listOwnership(HOSTNAME, EVENT_HUB_NAME, CONSUMER_GROUP)) .expectNextCount(1).verifyComplete(); - StepVerifier.create(checkpointStore.listOwnership(fullyQualifiedNamespace, eventHubName, consumerGroup)) + StepVerifier.create(checkpointStore.listOwnership(HOSTNAME, EVENT_HUB_NAME, CONSUMER_GROUP)) .assertNext(partitionOwnership -> { - assertEquals("1", partitionOwnership.getPartitionId(), "Partition"); - assertEquals(consumerGroup, partitionOwnership.getConsumerGroup(), "Consumer"); - assertEquals(eventHubName, partitionOwnership.getEventHubName(), "EventHub name"); + assertEquals(PARTITION_ID, partitionOwnership.getPartitionId(), "Partition"); + assertEquals(CONSUMER_GROUP, partitionOwnership.getConsumerGroup(), "Consumer"); + assertEquals(EVENT_HUB_NAME, partitionOwnership.getEventHubName(), "EventHub name"); assertEquals(eventProcessorClient.getIdentifier(), partitionOwnership.getOwnerId(), "OwnerId"); assertTrue(partitionOwnership.getLastModifiedTime() >= beforeTest, "LastModifiedTime"); assertTrue(partitionOwnership.getLastModifiedTime() <= System.currentTimeMillis(), "LastModifiedTime"); @@ -201,11 +206,11 @@ public void testWithSimplePartitionProcessor() throws Exception { eventProcessorClient.stop(); - StepVerifier.create(checkpointStore.listOwnership(fullyQualifiedNamespace, eventHubName, consumerGroup)) + StepVerifier.create(checkpointStore.listOwnership(HOSTNAME, EVENT_HUB_NAME, CONSUMER_GROUP)) .assertNext(partitionOwnership -> { - assertEquals("1", partitionOwnership.getPartitionId(), "Partition"); - assertEquals(consumerGroup, partitionOwnership.getConsumerGroup(), "Consumer"); - assertEquals(eventHubName, partitionOwnership.getEventHubName(), "EventHub name"); + assertEquals(PARTITION_ID, partitionOwnership.getPartitionId(), "Partition"); + assertEquals(CONSUMER_GROUP, partitionOwnership.getConsumerGroup(), "Consumer"); + assertEquals(EVENT_HUB_NAME, partitionOwnership.getEventHubName(), "EventHub name"); assertEquals("", partitionOwnership.getOwnerId(), "Owner Id"); assertTrue(partitionOwnership.getLastModifiedTime() >= beforeTest, "LastModifiedTime"); assertTrue(partitionOwnership.getLastModifiedTime() <= System.currentTimeMillis(), "LastModifiedTime"); @@ -220,16 +225,18 @@ public void testWithSimplePartitionProcessor() throws Exception { */ @Test @SuppressWarnings("unchecked") - public void testProcessSpans() throws Exception { + public void testProcessSpansAndMetrics() throws Exception { //Arrange - final Tracer tracer1 = mock(Tracer.class); - when(tracer1.isEnabled()).thenReturn(true); + final Tracer tracer = mock(Tracer.class); + final TestMeter meter = new TestMeter(); + when(tracer.isEnabled()).thenReturn(true); when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); - when(eventHubClientBuilder.createTracer()).thenReturn(tracer1); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn("test-ns"); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubClientBuilder.createTracer()).thenReturn(tracer); + when(eventHubClientBuilder.createMeter()).thenReturn(meter); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient .createConsumer(anyString(), anyInt(), eq(true))) .thenReturn(consumer1); @@ -246,30 +253,39 @@ public void testProcessSpans() throws Exception { when(eventData1.getProperties()).thenReturn(properties); when(consumer1.receiveFromPartition(anyString(), any(EventPosition.class), any(ReceiveOptions.class))) .thenReturn(Flux.just(getEvent(eventData1))); - when(tracer1.extractContext(any())).thenAnswer(invocation -> { + when(tracer.extractContext(any())).thenAnswer(invocation -> { Function consumer = invocation.getArgument(0, Function.class); assertEquals(diagnosticId, consumer.apply("traceparent")); assertNull(consumer.apply("tracestate")); return new Context(SPAN_CONTEXT_KEY, "value"); }); - when(tracer1.start(eq("EventHubs.process"), any(StartSpanOptions.class), any(Context.class))).thenAnswer( + final String expectedProcessSpanName = getSpanName(PROCESS, EVENT_HUB_NAME); + when(tracer.start(eq(expectedProcessSpanName), any(StartSpanOptions.class), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), 0); + assertStartOptions(PROCESS, invocation.getArgument(1, StartSpanOptions.class), 1); return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "value2"); } ); - CountDownLatch latch = new CountDownLatch(1); - when(tracer1.makeSpanCurrent(any())).thenReturn(() -> { }); + final String expectedSettleSpanName = getSpanName(CHECKPOINT, EVENT_HUB_NAME); + when(tracer.start(eq(expectedSettleSpanName), any(StartSpanOptions.class), any(Context.class))).thenAnswer( + invocation -> { + assertCheckpointStartOptions(invocation.getArgument(1, StartSpanOptions.class)); + return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "value2"); + } + ); + when(tracer.makeSpanCurrent(any())).thenReturn(() -> { }); + // processor span ends after TestPartitionProcessor latch counts down + CountDownLatch latch = new CountDownLatch(2); doAnswer(invocation -> { latch.countDown(); return null; - }).when(tracer1).end(isNull(), isNull(), any()); + }).when(tracer).end(any(), any(), any()); final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); - processorOptions.setConsumerGroup("test-consumer") + processorOptions.setConsumerGroup(CONSUMER_GROUP) .setTrackLastEnqueuedEventProperties(false) .setInitialEventPositionProvider(null) .setMaxBatchSize(1) @@ -281,7 +297,7 @@ public void testProcessSpans() throws Exception { //Act EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - TestPartitionProcessor::new, checkpointStore, EventProcessorClientTest::noopConsumer, tracer1, + TestPartitionProcessor::new, checkpointStore, EventProcessorClientTest::noopConsumer, tracer, meter, processorOptions); eventProcessorClient.start(); @@ -289,11 +305,12 @@ public void testProcessSpans() throws Exception { eventProcessorClient.stop(); //Assert - verify(tracer1, times(1)).extractContext(any()); - verify(tracer1, times(1)).start(eq("EventHubs.process"), any(), any(Context.class)); - verify(tracer1, times(1)).end(isNull(), isNull(), any()); - } + verify(tracer, times(1)).extractContext(any()); + verify(tracer, times(1)).start(eq(expectedProcessSpanName), any(), any(Context.class)); + verify(tracer, times(1)).start(eq(expectedSettleSpanName), any(), any(Context.class)); + assertProcessMetrics(meter, 1, null); + } /** * Tests process start spans invoked for {@link EventProcessorClient}. @@ -302,16 +319,18 @@ public void testProcessSpans() throws Exception { */ @Test @SuppressWarnings("unchecked") - public void testProcessBatchSpans() throws Exception { + public void testProcessBatchTracesAndMetrics() throws Exception { //Arrange - final Tracer tracer1 = mock(Tracer.class); - when(tracer1.isEnabled()).thenReturn(true); + final Tracer tracer = mock(Tracer.class); + final TestMeter meter = new TestMeter(); + when(tracer.isEnabled()).thenReturn(true); when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); - when(eventHubClientBuilder.createTracer()).thenReturn(tracer1); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn("test-ns"); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubClientBuilder.createTracer()).thenReturn(tracer); + when(eventHubClientBuilder.createMeter()).thenReturn(meter); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient .createConsumer(anyString(), anyInt(), eq(true))) .thenReturn(consumer1); @@ -335,7 +354,7 @@ public void testProcessBatchSpans() throws Exception { .thenReturn(Flux.just(getEvent(eventData1), getEvent(eventData2))); AtomicInteger counter = new AtomicInteger(0); - when(tracer1.extractContext(any())).thenAnswer(invocation -> { + when(tracer.extractContext(any())).thenAnswer(invocation -> { Function consumer = invocation.getArgument(0, Function.class); String traceparent = consumer.apply("traceparent"); if (counter.getAndIncrement() == 0) { @@ -345,20 +364,31 @@ public void testProcessBatchSpans() throws Exception { } return new Context(SPAN_CONTEXT_KEY, traceparent); }); - when(tracer1.start(eq("EventHubs.process"), any(StartSpanOptions.class), any(Context.class))).thenAnswer( + + final String expectedProcessSpanName = getSpanName(PROCESS, EVENT_HUB_NAME); + when(tracer.start(eq(expectedProcessSpanName), any(StartSpanOptions.class), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), 2); + assertStartOptions(PROCESS, invocation.getArgument(1, StartSpanOptions.class), 2); return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "value2"); } ); - CountDownLatch latch = new CountDownLatch(1); - when(tracer1.makeSpanCurrent(any())).thenReturn(() -> { }); + final String expectedCheckpointSpanName = getSpanName(CHECKPOINT, EVENT_HUB_NAME); + when(tracer.start(eq(expectedCheckpointSpanName), any(StartSpanOptions.class), any(Context.class))).thenAnswer( + invocation -> { + assertCheckpointStartOptions(invocation.getArgument(1, StartSpanOptions.class)); + return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "value3"); + } + ); + + when(tracer.makeSpanCurrent(any())).thenReturn(() -> { }); + // processor span ends after TestPartitionProcessor latch counts down + CountDownLatch latch = new CountDownLatch(2); doAnswer(invocation -> { latch.countDown(); return null; - }).when(tracer1).end(isNull(), isNull(), any()); + }).when(tracer).end(any(), any(), any()); final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); @@ -374,7 +404,95 @@ public void testProcessBatchSpans() throws Exception { //Act EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - TestPartitionProcessor::new, checkpointStore, EventProcessorClientTest::noopConsumer, tracer1, + TestPartitionProcessor::new, checkpointStore, EventProcessorClientTest::noopConsumer, tracer, meter, + processorOptions); + + eventProcessorClient.start(); + assertTrue(latch.await(10, TimeUnit.SECONDS)); + eventProcessorClient.stop(); + + //Assert + verify(tracer, times(2)).extractContext(any()); + verify(tracer, times(1)).start(eq(expectedProcessSpanName), any(), any(Context.class)); + verify(tracer, times(1)).start(eq(expectedCheckpointSpanName), any(), any(Context.class)); + + assertProcessMetrics(meter, 2, null); + } + + public static Stream errorSource() { + Throwable inner = new RuntimeException("test"); + final ArrayList arguments = new ArrayList<>(); + arguments.add(Arguments.of(inner, inner.getClass().getName())); + arguments.add(Arguments.of(Exceptions.propagate(inner), inner.getClass().getName())); + return arguments.stream(); + } + + @ParameterizedTest + @MethodSource("errorSource") + @SuppressWarnings("unchecked") + public void testProcessWithErrorTracesAndMetrics(RuntimeException error, String expectedErrorType) throws Exception { + //Arrange + final Tracer tracer = mock(Tracer.class); + final TestMeter meter = new TestMeter(); + when(tracer.isEnabled()).thenReturn(true); + when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); + when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); + when(eventHubClientBuilder.createTracer()).thenReturn(tracer); + when(eventHubClientBuilder.createMeter()).thenReturn(meter); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); + when(eventHubAsyncClient + .createConsumer(anyString(), anyInt(), eq(true))) + .thenReturn(consumer1); + when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); + when(eventData1.getSequenceNumber()).thenReturn(1L); + when(eventData1.getOffset()).thenReturn(100L); + when(eventData1.getEnqueuedTime()).thenReturn(Instant.ofEpochSecond(1560639208)); + when(eventData2.getEnqueuedTime()).thenReturn(Instant.ofEpochSecond(1560639209)); + + String diagnosticId1 = "00-08ee063508037b1719dddcbf248e30e2-1365c684eb25daed-01"; + Map properties1 = new HashMap<>(); + properties1.put(DIAGNOSTIC_ID_KEY, diagnosticId1); + + String diagnosticId2 = "00-18ee063508037b1719dddcbf248e30e2-1365c684eb25daed-01"; + Map properties2 = new HashMap<>(); + properties2.put(DIAGNOSTIC_ID_KEY, diagnosticId2); + + when(eventData1.getProperties()).thenReturn(properties1); + when(eventData2.getProperties()).thenReturn(properties2); + when(consumer1.receiveFromPartition(anyString(), any(EventPosition.class), any(ReceiveOptions.class))) + .thenReturn(Flux.just(getEvent(eventData1), getEvent(eventData2))); + + when(tracer.extractContext(any())).thenReturn(Context.NONE); + + final String expectedProcessSpanName = getSpanName(PROCESS, EVENT_HUB_NAME); + when(tracer.start(eq(expectedProcessSpanName), any(StartSpanOptions.class), any(Context.class))) + .thenReturn(new Context(PARENT_TRACE_CONTEXT_KEY, "span")); + when(tracer.makeSpanCurrent(any())).thenReturn(() -> { }); + + // processor span ends after TestPartitionProcessor latch counts down + CountDownLatch latch = new CountDownLatch(1); + doAnswer(invocation -> { + latch.countDown(); + return null; + }).when(tracer).end(any(), any(), any()); + + final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); + final TestPartitionProcessor processor = new TestPartitionProcessor(new CountDownLatch(1), error); + processorOptions.setConsumerGroup("test-consumer") + .setTrackLastEnqueuedEventProperties(false) + .setInitialEventPositionProvider(null) + .setMaxBatchSize(2) + .setMaxWaitTime(null) + .setBatchReceiveMode(true) + .setLoadBalancerUpdateInterval(Duration.ofSeconds(10)) + .setPartitionOwnershipExpirationInterval(Duration.ofMinutes(1)) + .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); + + //Act + EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, + () -> processor, checkpointStore, EventProcessorClientTest::noopConsumer, tracer, meter, processorOptions); eventProcessorClient.start(); @@ -382,9 +500,66 @@ public void testProcessBatchSpans() throws Exception { eventProcessorClient.stop(); //Assert - verify(tracer1, times(2)).extractContext(any()); - verify(tracer1, times(1)).start(eq("EventHubs.process"), any(), any(Context.class)); - verify(tracer1, times(1)).end(isNull(), isNull(), any()); + verify(tracer, times(2)).extractContext(any()); + verify(tracer, times(1)).start(eq(expectedProcessSpanName), any(), any(Context.class)); + verify(tracer, times(1)).end(eq(expectedErrorType), same(error), any(Context.class)); + + assertProcessMetrics(meter, 2, expectedErrorType); + } + + @Test + @SuppressWarnings("unchecked") + public void testTracingMetricsEmptyBatch() throws Exception { + //Arrange + final Tracer tracer = mock(Tracer.class); + final TestMeter meter = new TestMeter(); + when(tracer.isEnabled()).thenReturn(true); + when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); + when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); + when(eventHubClientBuilder.createTracer()).thenReturn(tracer); + when(eventHubClientBuilder.createMeter()).thenReturn(meter); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); + when(eventHubAsyncClient + .createConsumer(anyString(), anyInt(), eq(true))) + .thenReturn(consumer1); + when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); + when(consumer1.receiveFromPartition(anyString(), any(EventPosition.class), any(ReceiveOptions.class))) + .thenReturn(Flux.generate(sync -> { })); + when(tracer.makeSpanCurrent(any())).thenReturn(() -> { }); + // processor span ends after TestPartitionProcessor latch counts down + CountDownLatch latch = new CountDownLatch(1); + doAnswer(invocation -> { + latch.countDown(); + return null; + }).when(tracer).end(any(), any(), any()); + + final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); + + processorOptions.setConsumerGroup("test-consumer") + .setTrackLastEnqueuedEventProperties(false) + .setInitialEventPositionProvider(null) + .setMaxBatchSize(2) + .setMaxWaitTime(Duration.ofMillis(1)) + .setBatchReceiveMode(true) + .setLoadBalancerUpdateInterval(Duration.ofSeconds(10)) + .setPartitionOwnershipExpirationInterval(Duration.ofMinutes(1)) + .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); + + //Act + EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, + TestPartitionProcessor::new, checkpointStore, EventProcessorClientTest::noopConsumer, tracer, meter, + processorOptions); + + eventProcessorClient.start(); + assertFalse(latch.await(2, TimeUnit.SECONDS)); + eventProcessorClient.stop(); + + //Assert + verify(tracer, never()).start(anyString(), any(), any(Context.class)); + assertEquals(0, meter.getCounters().get("messaging.client.consumed.messages").getMeasurements().size()); + assertEquals(0, meter.getHistograms().get("messaging.process.duration").getMeasurements().size()); } /** @@ -400,9 +575,9 @@ public void testProcessSpansWithoutDiagnosticId() throws Exception { when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); when(eventHubClientBuilder.createTracer()).thenReturn(tracer); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn("test-ns"); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); when(eventHubAsyncClient .createConsumer(anyString(), anyInt(), eq(true))) @@ -424,23 +599,30 @@ public void testProcessSpansWithoutDiagnosticId() throws Exception { when(consumer1.receiveFromPartition(anyString(), any(EventPosition.class), any(ReceiveOptions.class))) .thenReturn(Flux.just(getEvent(eventData1), getEvent(eventData2), getEvent(eventData3))); - when(tracer.start(eq("EventHubs.process"), any(StartSpanOptions.class), any(Context.class))).thenAnswer( + final String expectedProcessSpanName = getSpanName(PROCESS, EVENT_HUB_NAME); + when(tracer.start(eq(expectedProcessSpanName), any(StartSpanOptions.class), any(Context.class))).thenAnswer( invocation -> { - assertStartOptions(invocation.getArgument(1, StartSpanOptions.class), 0); + assertStartOptions(PROCESS, invocation.getArgument(1, StartSpanOptions.class), 1); return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "value2"); } ); + final String expectedSettleSpanName = getSpanName(CHECKPOINT, EVENT_HUB_NAME); + when(tracer.start(eq(expectedSettleSpanName), any(StartSpanOptions.class), any(Context.class))).thenAnswer( + invocation -> { + assertCheckpointStartOptions(invocation.getArgument(1, StartSpanOptions.class)); + return invocation.getArgument(2, Context.class).addData(PARENT_TRACE_CONTEXT_KEY, "value2"); + } + ); + AtomicBoolean closed = new AtomicBoolean(false); when(tracer.makeSpanCurrent(any())).thenReturn(() -> closed.set(true)); final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); - CountDownLatch countDownLatch = new CountDownLatch(numberOfEvents); - TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(); - testPartitionProcessor.countDownLatch = countDownLatch; + TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(numberOfEvents); - processorOptions.setConsumerGroup("test-consumer") + processorOptions.setConsumerGroup(CONSUMER_GROUP) .setTrackLastEnqueuedEventProperties(false) .setInitialEventPositionProvider(null) .setMaxBatchSize(1) @@ -452,19 +634,22 @@ public void testProcessSpansWithoutDiagnosticId() throws Exception { //Act EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, tracer, + () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, tracer, null, processorOptions); eventProcessorClient.start(); - boolean success = countDownLatch.await(10, TimeUnit.SECONDS); + boolean success = testPartitionProcessor.countDownLatch.await(10, TimeUnit.SECONDS); eventProcessorClient.stop(); assertTrue(success); assertTrue(closed.get()); - // This is one less because the processEvent is called before the end span call, so it is possible for - // to reach this line without calling it the 5th time yet. (Timing issue.) - verify(tracer, times(numberOfEvents)).start(eq("EventHubs.process"), any(), any(Context.class)); + verify(tracer, times(numberOfEvents)).start(eq(expectedProcessSpanName), any(), any(Context.class)); + + // This is one less because the latch happens at the start of process callback + // and checkpoint/process spans are reported after + + verify(tracer, atLeast(numberOfEvents - 1)).start(eq(expectedSettleSpanName), any(), any(Context.class)); verify(tracer, atLeast(numberOfEvents - 1)).end(isNull(), isNull(), any()); } @@ -487,13 +672,13 @@ public void testWithMultiplePartitions() throws Exception { when(eventHubClientBuilder.getPrefetchCount()).thenReturn(EventHubClientBuilder.DEFAULT_PREFETCH_COUNT); when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1", "2", "3")); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn("test-ns"); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); when(eventHubAsyncClient.createConsumer(anyString(), eq(EventHubClientBuilder.DEFAULT_PREFETCH_COUNT), eq(true))) .thenReturn(consumer1, consumer2, consumer3); when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.fromIterable(identifiers)); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); when(consumer1.receiveFromPartition(argThat(arg -> identifiers.remove(arg)), eq(position), any())) @@ -516,7 +701,7 @@ public void testWithMultiplePartitions() throws Exception { final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); - processorOptions.setConsumerGroup("test-consumer") + processorOptions.setConsumerGroup(CONSUMER_GROUP) .setTrackLastEnqueuedEventProperties(false) .setInitialEventPositionProvider(null) .setMaxBatchSize(1) @@ -528,7 +713,7 @@ public void testWithMultiplePartitions() throws Exception { // Act final EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - TestPartitionProcessor::new, checkpointStore, EventProcessorClientTest::noopConsumer, null, + TestPartitionProcessor::new, checkpointStore, EventProcessorClientTest::noopConsumer, null, null, processorOptions); eventProcessorClient.start(); @@ -537,7 +722,7 @@ public void testWithMultiplePartitions() throws Exception { // Assert Assertions.assertTrue(completed); - StepVerifier.create(checkpointStore.listOwnership("test-ns", "test-eh", "test-consumer")) + StepVerifier.create(checkpointStore.listOwnership(HOSTNAME, EVENT_HUB_NAME, CONSUMER_GROUP)) .expectNextCount(1).verifyComplete(); verify(eventHubAsyncClient, atLeast(1)).getPartitionIds(); @@ -547,7 +732,7 @@ public void testWithMultiplePartitions() throws Exception { // We expected one to be removed. Assertions.assertEquals(2, identifiers.size()); - StepVerifier.create(checkpointStore.listOwnership("test-ns", "test-eh", "test-consumer")) + StepVerifier.create(checkpointStore.listOwnership(HOSTNAME, EVENT_HUB_NAME, CONSUMER_GROUP)) .assertNext(po -> { String partitionId = po.getPartitionId(); verify(consumer1, atLeastOnce()).receiveFromPartition(eq(partitionId), any(EventPosition.class), any()); @@ -558,15 +743,13 @@ public void testWithMultiplePartitions() throws Exception { public void testPrefetchCountSet() throws Exception { // Arrange final String consumerGroup = "my-consumer-group"; - final String eventHubName = "test-event-hub"; - final String fullyQualifiedNamespace = "test-namespace"; final int prefetch = 15; when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); when(eventHubClientBuilder.getPrefetchCount()).thenReturn(prefetch); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(fullyQualifiedNamespace); - when(eventHubAsyncClient.getEventHubName()).thenReturn(eventHubName); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); when(eventHubAsyncClient .createConsumer(eq(consumerGroup), eq(prefetch), eq(true))) @@ -581,9 +764,7 @@ public void testPrefetchCountSet() throws Exception { when(eventData3.getOffset()).thenReturn(150L); final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); - final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(); - CountDownLatch countDownLatch = new CountDownLatch(3); - testPartitionProcessor.countDownLatch = countDownLatch; + final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(3); processorOptions.setConsumerGroup(consumerGroup) .setTrackLastEnqueuedEventProperties(false) @@ -596,12 +777,12 @@ public void testPrefetchCountSet() throws Exception { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, + () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, null, processorOptions); // Act eventProcessorClient.start(); - boolean completed = countDownLatch.await(10, TimeUnit.SECONDS); + boolean completed = testPartitionProcessor.countDownLatch.await(10, TimeUnit.SECONDS); eventProcessorClient.stop(); // Assert @@ -618,9 +799,9 @@ public void testDefaultPrefetch() throws Exception { when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); when(eventHubClientBuilder.getPrefetchCount()).thenReturn(null); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn("test-ns"); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); when(eventHubAsyncClient .createConsumer(eq(consumerGroup), eq(EventHubClientBuilder.DEFAULT_PREFETCH_COUNT), eq(true))) @@ -635,9 +816,7 @@ public void testDefaultPrefetch() throws Exception { when(eventData3.getOffset()).thenReturn(150L); final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); - final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(); - CountDownLatch countDownLatch = new CountDownLatch(3); - testPartitionProcessor.countDownLatch = countDownLatch; + final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(3); processorOptions.setConsumerGroup(consumerGroup) .setTrackLastEnqueuedEventProperties(false) @@ -650,12 +829,12 @@ public void testDefaultPrefetch() throws Exception { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, + () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, null, processorOptions); // Act eventProcessorClient.start(); - boolean completed = countDownLatch.await(10, TimeUnit.SECONDS); + boolean completed = testPartitionProcessor.countDownLatch.await(10, TimeUnit.SECONDS); eventProcessorClient.stop(); // Assert @@ -670,9 +849,9 @@ public void testBatchReceive() throws Exception { // Arrange when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn("test-ns"); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); when(eventHubAsyncClient .createConsumer(anyString(), anyInt(), eq(true))) @@ -687,11 +866,9 @@ public void testBatchReceive() throws Exception { when(eventData3.getOffset()).thenReturn(150L); final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); - final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(); - CountDownLatch countDownLatch = new CountDownLatch(3); - testPartitionProcessor.countDownLatch = countDownLatch; + final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(3); - processorOptions.setConsumerGroup("test-consumer") + processorOptions.setConsumerGroup(CONSUMER_GROUP) .setTrackLastEnqueuedEventProperties(false) .setInitialEventPositionProvider(null) .setMaxBatchSize(2) @@ -702,12 +879,12 @@ public void testBatchReceive() throws Exception { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, + () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, null, processorOptions); // Act eventProcessorClient.start(); - boolean completed = countDownLatch.await(10, TimeUnit.SECONDS); + boolean completed = testPartitionProcessor.countDownLatch.await(10, TimeUnit.SECONDS); eventProcessorClient.stop(); // Assert @@ -720,9 +897,9 @@ public void testBatchReceiveHeartBeat() throws InterruptedException { // Arrange when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn("test-ns"); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); when(eventHubAsyncClient .createConsumer(anyString(), anyInt(), eq(true))) @@ -737,11 +914,9 @@ public void testBatchReceiveHeartBeat() throws InterruptedException { when(eventData3.getOffset()).thenReturn(150L); final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); - final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(); - CountDownLatch countDownLatch = new CountDownLatch(1); - testPartitionProcessor.countDownLatch = countDownLatch; + final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(1); - processorOptions.setConsumerGroup("test-consumer") + processorOptions.setConsumerGroup(CONSUMER_GROUP) .setTrackLastEnqueuedEventProperties(false) .setInitialEventPositionProvider(null) .setMaxBatchSize(2) @@ -752,12 +927,12 @@ public void testBatchReceiveHeartBeat() throws InterruptedException { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, + () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, null, processorOptions); // Act eventProcessorClient.start(); - boolean completed = countDownLatch.await(20, TimeUnit.SECONDS); + boolean completed = testPartitionProcessor.countDownLatch.await(20, TimeUnit.SECONDS); eventProcessorClient.stop(); // Assert @@ -770,13 +945,12 @@ public void testBatchReceiveHeartBeat() throws InterruptedException { @SuppressWarnings("unchecked") public void testSingleEventReceiveHeartBeat() throws InterruptedException { // Arrange - Tracer tracer = mock(Tracer.class); when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); when(eventHubClientBuilder.createTracer()).thenReturn(null); - when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn("test-ns"); - when(eventHubAsyncClient.getEventHubName()).thenReturn("test-eh"); - when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just("1")); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); when(eventHubAsyncClient .createConsumer(anyString(), anyInt(), eq(true))) @@ -796,23 +970,11 @@ public void testSingleEventReceiveHeartBeat() throws InterruptedException { when(eventData1.getProperties()).thenReturn(properties); when(eventData2.getProperties()).thenReturn(properties); - when(tracer.extractContext(any())).thenAnswer( - invocation -> { - Function getter = invocation.getArgument(0, Function.class); - assertEquals(diagnosticId, getter.apply(DIAGNOSTIC_ID_KEY)); - return invocation.getArgument(1, Context.class).addData(SPAN_CONTEXT_KEY, "value"); - } - ); - when(tracer.start(eq("EventHubs.process"), any(), any(Context.class))).thenAnswer( - invocation -> invocation.getArgument(1, Context.class) - .addData(PARENT_TRACE_CONTEXT_KEY, "value2")); final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); - final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(); - CountDownLatch countDownLatch = new CountDownLatch(1); - testPartitionProcessor.countDownLatch = countDownLatch; + final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(1); - processorOptions.setConsumerGroup("test-consumer") + processorOptions.setConsumerGroup(CONSUMER_GROUP) .setTrackLastEnqueuedEventProperties(false) .setInitialEventPositionProvider(null) .setMaxBatchSize(1) @@ -823,11 +985,67 @@ public void testSingleEventReceiveHeartBeat() throws InterruptedException { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, - () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, + () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, null, processorOptions); eventProcessorClient.start(); - boolean completed = countDownLatch.await(20, TimeUnit.SECONDS); + boolean completed = testPartitionProcessor.countDownLatch.await(20, TimeUnit.SECONDS); + eventProcessorClient.stop(); + assertTrue(completed); + assertTrue(testPartitionProcessor.receivedEventsCount.contains(0)); + assertTrue(testPartitionProcessor.receivedEventsCount.contains(1)); + } + + + @Test + @SuppressWarnings("unchecked") + public void passesInstrumentedCheckpointStore() throws InterruptedException { + // Arrange + when(eventHubClientBuilder.getPrefetchCount()).thenReturn(DEFAULT_PREFETCH_COUNT); + when(eventHubClientBuilder.buildAsyncClient()).thenReturn(eventHubAsyncClient); + when(eventHubClientBuilder.createTracer()).thenReturn(null); + when(eventHubAsyncClient.getFullyQualifiedNamespace()).thenReturn(HOSTNAME); + when(eventHubAsyncClient.getEventHubName()).thenReturn(EVENT_HUB_NAME); + when(eventHubAsyncClient.getPartitionIds()).thenReturn(Flux.just(PARTITION_ID)); + when(eventHubAsyncClient.getIdentifier()).thenReturn("my-client-identifier"); + when(eventHubAsyncClient + .createConsumer(anyString(), anyInt(), eq(true))) + .thenReturn(consumer1); + when(consumer1.receiveFromPartition(anyString(), any(EventPosition.class), any(ReceiveOptions.class))) + .thenReturn(Flux.just(getEvent(eventData1), getEvent(eventData2)).delayElements(Duration.ofSeconds(3))); + when(eventData1.getSequenceNumber()).thenReturn(1L); + when(eventData1.getOffset()).thenReturn(1L); + when(eventData1.getEnqueuedTime()).thenReturn(Instant.ofEpochSecond(1560639208)); + when(eventData2.getSequenceNumber()).thenReturn(2L); + when(eventData2.getOffset()).thenReturn(100L); + when(eventData2.getEnqueuedTime()).thenReturn(Instant.ofEpochSecond(1560639208)); + + String diagnosticId = "00-08ee063508037b1719dddcbf248e30e2-1365c684eb25daed-01"; + Map properties = new HashMap<>(); + properties.put(DIAGNOSTIC_ID_KEY, diagnosticId); + + when(eventData1.getProperties()).thenReturn(properties); + when(eventData2.getProperties()).thenReturn(properties); + + final SampleCheckpointStore checkpointStore = new SampleCheckpointStore(); + final TestPartitionProcessor testPartitionProcessor = new TestPartitionProcessor(1); + + processorOptions.setConsumerGroup(CONSUMER_GROUP) + .setTrackLastEnqueuedEventProperties(false) + .setInitialEventPositionProvider(null) + .setMaxBatchSize(1) + .setMaxWaitTime(Duration.ofSeconds(1)) + .setBatchReceiveMode(false) + .setLoadBalancerUpdateInterval(Duration.ofSeconds(10)) + .setPartitionOwnershipExpirationInterval(Duration.ofMinutes(1)) + .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); + + final EventProcessorClient eventProcessorClient = new EventProcessorClient(eventHubClientBuilder, + () -> testPartitionProcessor, checkpointStore, EventProcessorClientTest::noopConsumer, null, null, + processorOptions); + + eventProcessorClient.start(); + boolean completed = testPartitionProcessor.countDownLatch.await(20, TimeUnit.SECONDS); eventProcessorClient.stop(); assertTrue(completed); assertTrue(testPartitionProcessor.receivedEventsCount.contains(0)); @@ -838,29 +1056,79 @@ static void noopConsumer(ErrorContext unused) { } private PartitionEvent getEvent(EventData event) { - PartitionContext context = new PartitionContext("test-ns", "foo", "bar", "baz"); + PartitionContext context = new PartitionContext(HOSTNAME, EVENT_HUB_NAME, CONSUMER_GROUP, PARTITION_ID); return new PartitionEvent(context, event, null); } - private void assertStartOptions(StartSpanOptions startOpts, int linkCount) { + private void assertStartOptions(OperationName operationName, StartSpanOptions startOpts, int linkCount) { assertEquals(SpanKind.CONSUMER, startOpts.getSpanKind()); - assertEquals("test-eh", startOpts.getAttributes().get(ENTITY_PATH_KEY)); - assertEquals("test-ns", startOpts.getAttributes().get(HOST_NAME_KEY)); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, PARTITION_ID, CONSUMER_GROUP, null, + operationName, startOpts.getAttributes()); if (linkCount == 0) { - assertTrue(startOpts.getAttributes().containsKey(MESSAGE_ENQUEUED_TIME_ATTRIBUTE_NAME)); + assertTrue(startOpts.getAttributes().containsKey(MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME)); assertNull(startOpts.getLinks()); } else { assertEquals(linkCount, startOpts.getLinks().size()); - for (TracingLink link : startOpts.getLinks()) { - assertTrue(link.getAttributes().containsKey(MESSAGE_ENQUEUED_TIME_ATTRIBUTE_NAME)); + if (linkCount == 1) { + assertTrue(startOpts.getAttributes().containsKey(MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME)); + } else { + for (TracingLink link : startOpts.getLinks()) { + assertTrue(link.getAttributes().containsKey(MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME)); + } } } } + private void assertCheckpointStartOptions(StartSpanOptions startOpts) { + assertEquals(SpanKind.INTERNAL, startOpts.getSpanKind()); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, PARTITION_ID, CONSUMER_GROUP, null, + CHECKPOINT, startOpts.getAttributes()); + assertNull(startOpts.getLinks()); + } + + private static void assertProcessMetrics(TestMeter meter, int batchSize, String expectedErrorType) { + TestCounter eventCounter = meter.getCounters().get("messaging.client.consumed.messages"); + assertNotNull(eventCounter); + assertEquals(1, eventCounter.getMeasurements().size()); + assertEquals(batchSize, eventCounter.getMeasurements().get(0).getValue()); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, PARTITION_ID, CONSUMER_GROUP, expectedErrorType, + PROCESS, eventCounter.getMeasurements().get(0).getAttributes()); + + TestHistogram processDuration = meter.getHistograms().get("messaging.process.duration"); + assertNotNull(processDuration); + assertEquals(1, processDuration.getMeasurements().size()); + assertNotNull(processDuration.getMeasurements().get(0).getValue()); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, PARTITION_ID, CONSUMER_GROUP, expectedErrorType, + PROCESS, processDuration.getMeasurements().get(0).getAttributes()); + + if (expectedErrorType == null) { + TestHistogram checkpointDuration = meter.getHistograms().get("messaging.client.operation.duration"); + assertNotNull(checkpointDuration); + assertEquals(1, checkpointDuration.getMeasurements().size()); + assertNotNull(checkpointDuration.getMeasurements().get(0).getValue()); + assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, PARTITION_ID, CONSUMER_GROUP, null, + CHECKPOINT, checkpointDuration.getMeasurements().get(0).getAttributes()); + } + } + private static final class TestPartitionProcessor extends PartitionProcessor { List receivedEventsCount = new ArrayList<>(); - CountDownLatch countDownLatch; + final CountDownLatch countDownLatch; + private final RuntimeException error; + + TestPartitionProcessor() { + this(null, null); + } + + TestPartitionProcessor(int count) { + this(new CountDownLatch(count), null); + } + + TestPartitionProcessor(CountDownLatch countDownLatch, RuntimeException error) { + this.countDownLatch = countDownLatch; + this.error = error; + } @Override public void processEvent(EventContext eventContext) { @@ -868,8 +1136,13 @@ public void processEvent(EventContext eventContext) { receivedEventsCount.add(1); if (countDownLatch != null) { countDownLatch.countDown(); - eventContext.updateCheckpoint(); + + if (error != null) { + throw error; + } } + + eventContext.updateCheckpoint(); } else { receivedEventsCount.add(0); } @@ -883,6 +1156,11 @@ public void processEventBatch(EventBatchContext eventBatchContext) { countDownLatch.countDown(); } }); + + if (error != null) { + throw error; + } + eventBatchContext.updateCheckpoint(); } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java index ceb3fd8e808aa..29072d6a6fe4e 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java @@ -52,7 +52,10 @@ public abstract class IntegrationTestBase extends TestBase { // Tests use timeouts of 20-60 seconds to verify something has happened // We need a short try timeout so that if transient issue happens we have a chance to retry it before overall test timeout. // This is a good idea to do in any production application as well - no point in waiting too long - protected static final AmqpRetryOptions RETRY_OPTIONS = new AmqpRetryOptions().setTryTimeout(Duration.ofSeconds(3)); + protected static final AmqpRetryOptions RETRY_OPTIONS = new AmqpRetryOptions() + .setTryTimeout(Duration.ofSeconds(3)) + .setMaxDelay(Duration.ofSeconds(3)) + .setMaxRetries(10); protected final ClientLogger logger; diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/PartitionBasedLoadBalancerTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/PartitionBasedLoadBalancerTest.java index 819ce13528614..b07a18647dbc6 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/PartitionBasedLoadBalancerTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/PartitionBasedLoadBalancerTest.java @@ -7,7 +7,7 @@ import com.azure.core.util.logging.ClientLogger; import com.azure.core.util.logging.LogLevel; import com.azure.messaging.eventhubs.implementation.PartitionProcessor; -import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer; +import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; import com.azure.messaging.eventhubs.models.ErrorContext; import com.azure.messaging.eventhubs.models.EventBatchContext; import com.azure.messaging.eventhubs.models.EventContext; @@ -85,8 +85,8 @@ public class PartitionBasedLoadBalancerTest { private static final boolean BATCH_RECEIVE_MODE = false; private static final PartitionContext PARTITION_CONTEXT = new PartitionContext(FQ_NAMESPACE, EVENT_HUB_NAME, CONSUMER_GROUP_NAME, "bazz"); - private static final EventHubsTracer DEFAULT_TRACER = - new EventHubsTracer(null, FQ_NAMESPACE, EVENT_HUB_NAME); + private static final EventHubsConsumerInstrumentation DEFAULT_INSTRUMENTATION = new EventHubsConsumerInstrumentation(null, null, + FQ_NAMESPACE, EVENT_HUB_NAME, CONSUMER_GROUP_NAME, false); private static final String OWNER_ID_1 = "owner1"; private static final String OWNER_ID_2 = "owner2"; @@ -410,7 +410,7 @@ public void testReceiveFailure() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); PartitionPumpManager partitionPumpManager = new PartitionPumpManager(checkpointStore, - () -> partitionProcessor, eventHubClientBuilder, DEFAULT_TRACER, processorOptions); + () -> partitionProcessor, eventHubClientBuilder, DEFAULT_INSTRUMENTATION, processorOptions); PartitionBasedLoadBalancer loadBalancer = new PartitionBasedLoadBalancer(checkpointStore, eventHubAsyncClient, FQ_NAMESPACE, EVENT_HUB_NAME, CONSUMER_GROUP_NAME, "owner", TimeUnit.SECONDS.toSeconds(5), @@ -443,7 +443,7 @@ public void testCheckpointStoreListOwnershipFailure() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); PartitionPumpManager partitionPumpManager = new PartitionPumpManager(checkpointStore, () -> partitionProcessor, - eventHubClientBuilder, DEFAULT_TRACER, processorOptions); + eventHubClientBuilder, DEFAULT_INSTRUMENTATION, processorOptions); PartitionBasedLoadBalancer loadBalancer = new PartitionBasedLoadBalancer(checkpointStore, eventHubAsyncClient, FQ_NAMESPACE, EVENT_HUB_NAME, CONSUMER_GROUP_NAME, "owner", TimeUnit.SECONDS.toSeconds(5), @@ -510,7 +510,7 @@ public void testCheckpointStoreClaimOwnershipFailure() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager partitionPumpManager = new PartitionPumpManager(mockCheckpointStore, - () -> partitionProcessor, eventHubClientBuilder, DEFAULT_TRACER, processorOptions); + () -> partitionProcessor, eventHubClientBuilder, DEFAULT_INSTRUMENTATION, processorOptions); toClose.add(() -> partitionPumpManager.stopAllPartitionPumps().block()); final PartitionBasedLoadBalancer loadBalancer = new PartitionBasedLoadBalancer(mockCheckpointStore, @@ -552,7 +552,7 @@ public void testEventHubClientFailure() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); PartitionPumpManager partitionPumpManager = new PartitionPumpManager(checkpointStore, - () -> partitionProcessor, eventHubClientBuilder, DEFAULT_TRACER, processorOptions); + () -> partitionProcessor, eventHubClientBuilder, DEFAULT_INSTRUMENTATION, processorOptions); toClose.add(() -> partitionPumpManager.stopAllPartitionPumps().block()); PartitionBasedLoadBalancer loadBalancer = new PartitionBasedLoadBalancer(checkpointStore, @@ -834,7 +834,7 @@ public void processError(ErrorContext eventProcessingErrorContext) { eventProcessingErrorContext.getPartitionContext().getPartitionId(), eventProcessingErrorContext.getThrowable()); } - }, eventHubClientBuilder, DEFAULT_TRACER, processorOptions); + }, eventHubClientBuilder, DEFAULT_INSTRUMENTATION, processorOptions); toClose.add(() -> pumpManager.stopAllPartitionPumps().block()); diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/PartitionPumpManagerTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/PartitionPumpManagerTest.java index 87b7360e565b2..7a261713440ac 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/PartitionPumpManagerTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/PartitionPumpManagerTest.java @@ -5,7 +5,7 @@ import com.azure.messaging.eventhubs.implementation.PartitionProcessor; import com.azure.messaging.eventhubs.implementation.PartitionProcessorException; -import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer; +import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; import com.azure.messaging.eventhubs.models.Checkpoint; import com.azure.messaging.eventhubs.models.CloseContext; import com.azure.messaging.eventhubs.models.ErrorContext; @@ -72,8 +72,8 @@ public class PartitionPumpManagerTest { private static final String ETAG = "etag1"; private static final PartitionContext PARTITION_CONTEXT = new PartitionContext(FULLY_QUALIFIED_NAME, EVENTHUB_NAME, CONSUMER_GROUP, PARTITION_ID); - private static final EventHubsTracer DEFAULT_TRACER = - new EventHubsTracer(null, FULLY_QUALIFIED_NAME, EVENTHUB_NAME); + private static final EventHubsConsumerInstrumentation DEFAULT_INSTRUMENTATION = new EventHubsConsumerInstrumentation(null, null, + FULLY_QUALIFIED_NAME, EVENTHUB_NAME, CONSUMER_GROUP, false); @Mock private CheckpointStore checkpointStore; @Mock @@ -182,7 +182,7 @@ public void startPartitionPumpAtCorrectPosition(Long offset, Long sequenceNumber .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); try { // Act @@ -246,7 +246,7 @@ public void startPartitionPumpOnce() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); checkpoint.setOffset(1L).setSequenceNumber(10L); partitionOwnership.setLastModifiedTime(OffsetDateTime.now().toEpochSecond()); @@ -289,7 +289,7 @@ public void startPartitionPumpCleansUpOnError() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); final Exception testException = new IllegalStateException("Dummy exception."); when(consumerAsyncClient.receiveFromPartition( @@ -338,7 +338,7 @@ public void stopAllPartitionPumps() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); final String partition1 = "01"; final EventHubConsumerAsyncClient client1 = mock(EventHubConsumerAsyncClient.class); @@ -392,7 +392,7 @@ public void processesEventBatchWithLastEnqueued() throws InterruptedException { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Mock events to add. final Instant retrievalTime = Instant.now(); @@ -464,7 +464,7 @@ public void processBatchPrefetch(int maxBatchSize) throws InterruptedException { .setBatchReceiveMode(true); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, () -> partitionProcessor, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); final AtomicInteger publishedCounter = new AtomicInteger(); final Instant retrievalTime = Instant.now(); @@ -524,7 +524,7 @@ public void processBatchNotEnoughEventsAfterMaxTime() throws InterruptedExceptio .setBatchReceiveMode(true); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, () -> partitionProcessor, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); final Instant retrievalTime = Instant.now(); @@ -564,7 +564,7 @@ public void processBatchNotEnoughEventsNever() throws InterruptedException { .setBatchReceiveMode(true); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, () -> partitionProcessor, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); final Instant retrievalTime = Instant.now(); @@ -613,7 +613,7 @@ public void startPositionReturnsLatest() { final EventPosition expected = EventPosition.latest(); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Act final EventPosition actual = manager.getInitialEventPosition(partitionId, null); @@ -659,7 +659,7 @@ public void startPositionReturnsCheckpointOffset() { final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Act final EventPosition actual = manager.getInitialEventPosition(partitionId, checkpoint); @@ -700,7 +700,7 @@ public void startPositionReturnsCheckpointSequenceNumber() { final EventPosition expected = EventPosition.fromSequenceNumber(sequenceNumber); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Act final EventPosition actual = manager.getInitialEventPosition(partitionId, checkpoint); @@ -737,7 +737,7 @@ public void startPositionReturnsMapPosition() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Act final EventPosition actual = manager.getInitialEventPosition(partitionId, checkpoint); @@ -772,7 +772,7 @@ public void startPositionReturnsDefaultPosition() { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Act final EventPosition actual = manager.getInitialEventPosition(partitionId, checkpoint); @@ -806,7 +806,7 @@ public void processErrorCleansUpPartitionOnException() throws InterruptedExcepti .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Mock events to add. final EventData eventData1 = new EventData("1"); @@ -892,7 +892,7 @@ public void closeOnErrorCleansUpPartitionOnException() throws InterruptedExcepti .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Mock events to add. final EventData eventData1 = new EventData("1"); @@ -982,7 +982,7 @@ public void closeCleansUpPartitionOnException() throws InterruptedException { .setLoadBalancingStrategy(LoadBalancingStrategy.BALANCED); final PartitionPumpManager manager = new PartitionPumpManager(checkpointStore, supplier, builder, - DEFAULT_TRACER, options); + DEFAULT_INSTRUMENTATION, options); // Mock events to add. final EventData eventData1 = new EventData("1"); diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java new file mode 100644 index 0000000000000..0944d36bb7216 --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.core.util.logging.LoggingEventBuilder; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.data.SpanData; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestSpanProcessor implements SpanProcessor { + private static final ClientLogger LOGGER = new ClientLogger(TestSpanProcessor.class); + private final ConcurrentLinkedDeque spans = new ConcurrentLinkedDeque<>(); + private final String entityName; + private final String namespace; + private final String testName; + + private final AtomicReference> notifier = new AtomicReference<>(); + + public TestSpanProcessor(String namespace, String entityName, String testName) { + this.namespace = namespace; + this.entityName = entityName; + this.testName = testName; + } + + public List getEndedSpans() { + return new ArrayList<>(spans); + } + + @Override + public void onStart(Context context, ReadWriteSpan readWriteSpan) { + } + + @Override + public boolean isStartRequired() { + return false; + } + + @Override + public void onEnd(ReadableSpan readableSpan) { + SpanData span = readableSpan.toSpanData(); + + InstrumentationScopeInfo instrumentationScopeInfo = span.getInstrumentationScopeInfo(); + LoggingEventBuilder log = LOGGER.atInfo() + .addKeyValue("testName", testName) + .addKeyValue("name", span.getName()) + .addKeyValue("traceId", span.getTraceId()) + .addKeyValue("spanId", span.getSpanId()) + .addKeyValue("parentSpanId", span.getParentSpanId()) + .addKeyValue("kind", span.getKind()) + .addKeyValue("tracerName", instrumentationScopeInfo.getName()) + .addKeyValue("tracerVersion", instrumentationScopeInfo.getVersion()) + .addKeyValue("attributes", span.getAttributes()); + + for (int i = 0; i < span.getLinks().size(); i++) { + LinkData link = span.getLinks().get(i); + log.addKeyValue("linkTraceId" + i, link.getSpanContext().getTraceId()) + .addKeyValue("linkSpanId" + i, link.getSpanContext().getSpanId()) + .addKeyValue("linkAttributes" + i, link.getAttributes()); + } + log.log("got span"); + + spans.add(readableSpan); + Consumer filter = notifier.get(); + if (filter != null) { + filter.accept(readableSpan); + } + + // Various attribute keys can be found in: + // sdk/core/azure-core-metrics-opentelemetry/src/main/java/com/azure/core/metrics/opentelemetry/OpenTelemetryAttributes.java + // sdk/core/azure-core-tracing-opentelemetry/src/main/java/com/azure/core/tracing/opentelemetry/OpenTelemetryUtils.java + assertEquals("Microsoft.EventHub", readableSpan.getAttribute(AttributeKey.stringKey("az.namespace"))); + assertEquals("eventhubs", readableSpan.getAttribute(AttributeKey.stringKey("messaging.system"))); + assertEquals(entityName, readableSpan.getAttribute(AttributeKey.stringKey("messaging.destination.name"))); + assertEquals(namespace, readableSpan.getAttribute(AttributeKey.stringKey("server.address"))); + } + + public void notifyIfCondition(CountDownLatch countDownLatch, Predicate filter) { + notifier.set((span) -> { + if (filter.test(span)) { + LOGGER.atInfo() + .addKeyValue("traceId", span.getSpanContext().getTraceId()) + .addKeyValue("spanId", span.getSpanContext().getSpanId()) + .log("condition met"); + countDownLatch.countDown(); + } + }); + } + + @Override + public boolean isEndRequired() { + return true; + } + + @Override + public CompletableResultCode shutdown() { + notifier.set(null); + return CompletableResultCode.ofSuccess(); + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestUtils.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestUtils.java index d103c4f7a06ca..f072b9c935073 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestUtils.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestUtils.java @@ -13,7 +13,13 @@ import com.azure.core.util.logging.ClientLogger; import com.azure.core.util.logging.LogLevel; import com.azure.identity.AzurePipelinesCredentialBuilder; +import com.azure.core.amqp.models.AmqpAnnotatedMessage; +import com.azure.core.util.Context; +import com.azure.messaging.eventhubs.implementation.instrumentation.OperationName; import com.azure.messaging.eventhubs.models.PartitionEvent; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.trace.data.SpanData; import org.apache.qpid.proton.Proton; import org.apache.qpid.proton.amqp.Binary; import org.apache.qpid.proton.amqp.Symbol; @@ -54,6 +60,18 @@ import static com.azure.core.amqp.ProxyOptions.PROXY_USERNAME; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.ERROR_TYPE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_CONSUMER_GROUP_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_PARTITION_ID; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_NAME; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_OPERATION_TYPE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_SYSTEM; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_SYSTEM_VALUE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.SERVER_ADDRESS; +import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.getOperationType; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; /** * Contains helper methods for working with AMQP messages @@ -310,6 +328,58 @@ private static String getPropertyValue(String propertyName) { return Configuration.getGlobalConfiguration().get(propertyName, System.getenv(propertyName)); } + public static void assertAttributes(String hostname, String entityName, OperationName operationName, Map attributes) { + assertAllAttributes(hostname, entityName, null, null, null, operationName, attributes); + } + + public static Map attributesToMap(Attributes attributes) { + return attributes.asMap().entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey().getKey(), e -> e.getValue())); + } + + public static String getSpanName(OperationName operation, String eventHubName) { + return String.format("%s %s", operation, eventHubName); + } + + public static void assertAllAttributes(String hostname, String entityName, String partitionId, + String consumerGroup, String errorType, OperationName operationName, Map attributes) { + assertEquals(MESSAGING_SYSTEM_VALUE, attributes.get(MESSAGING_SYSTEM)); + assertEquals(hostname, attributes.get(SERVER_ADDRESS)); + assertEquals(entityName, attributes.get(MESSAGING_DESTINATION_NAME)); + assertEquals(partitionId, attributes.get(MESSAGING_DESTINATION_PARTITION_ID)); + assertEquals(consumerGroup, attributes.get(MESSAGING_CONSUMER_GROUP_NAME)); + if (operationName == null) { + assertNull(attributes.get(MESSAGING_OPERATION_NAME)); + assertNull(attributes.get(MESSAGING_OPERATION_TYPE)); + } else { + assertEquals(operationName.toString(), attributes.get(MESSAGING_OPERATION_NAME)); + assertEquals(getOperationType(operationName), attributes.get(MESSAGING_OPERATION_TYPE)); + } + assertEquals(errorType, attributes.get(ERROR_TYPE)); + } + + public static void assertSpanStatus(String description, SpanData span) { + if (description != null) { + assertEquals(StatusCode.ERROR, span.getStatus().getStatusCode()); + assertEquals(description, span.getStatus().getDescription()); + } else { + assertEquals(StatusCode.UNSET, span.getStatus().getStatusCode()); + } + } + + public static EventData createEventData(AmqpAnnotatedMessage amqpAnnotatedMessage, long offset, + long sequenceNumber, Instant enqueuedTime) { + amqpAnnotatedMessage.getMessageAnnotations() + .put(AmqpMessageConstant.OFFSET_ANNOTATION_NAME.getValue(), offset); + amqpAnnotatedMessage.getMessageAnnotations() + .put(AmqpMessageConstant.SEQUENCE_NUMBER_ANNOTATION_NAME.getValue(), sequenceNumber); + amqpAnnotatedMessage.getMessageAnnotations() + .put(AmqpMessageConstant.ENQUEUED_TIME_UTC_ANNOTATION_NAME.getValue(), enqueuedTime); + + SystemProperties systemProperties = new SystemProperties(amqpAnnotatedMessage, offset, enqueuedTime, sequenceNumber, null); + return new EventData(amqpAnnotatedMessage, systemProperties, Context.NONE); + } + private TestUtils() { } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index ded9f8cd8a89f..68e653d52bf32 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -8,7 +8,7 @@ import com.azure.core.util.ClientOptions; import com.azure.core.util.TracingOptions; import com.azure.core.util.logging.ClientLogger; -import com.azure.core.util.logging.LoggingEventBuilder; +import com.azure.messaging.eventhubs.implementation.instrumentation.OperationName; import com.azure.messaging.eventhubs.models.CreateBatchOptions; import com.azure.messaging.eventhubs.models.EventPosition; import com.azure.messaging.eventhubs.models.PartitionEvent; @@ -20,15 +20,10 @@ import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.context.Context; import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.common.InstrumentationScopeInfo; -import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.SpanProcessor; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.data.SpanData; import org.junit.jupiter.api.Disabled; @@ -48,15 +43,17 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.function.Predicate; import static com.azure.messaging.eventhubs.TestUtils.getEventHubName; import static com.azure.messaging.eventhubs.TestUtils.getFullyQualifiedDomainName; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.EVENT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.CHECKPOINT; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.PROCESS; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.RECEIVE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.SEND; import static java.util.stream.Collectors.toList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -75,7 +72,8 @@ public class TracingIntegrationTests extends IntegrationTestBase { private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); private final AtomicReference cachedCredential = new AtomicReference<>(); - + private static final AttributeKey OPERATION_NAME_ATTRIBUTE = AttributeKey.stringKey("messaging.operation.name"); + private static final AttributeKey OPERATION_TYPE_ATTRIBUTE = AttributeKey.stringKey("messaging.operation.type"); private TestSpanProcessor spanProcessor; private EventHubProducerAsyncClient producer; private EventHubConsumerAsyncClient consumer; @@ -140,7 +138,7 @@ public void sendAndReceiveFromPartition() throws InterruptedException { AtomicReference receivedSpan = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(2); - spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || span.getName().equals("EventHubs.send")); + spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer .receiveFromPartition(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime)) .take(1) @@ -154,18 +152,19 @@ public void sendAndReceiveFromPartition() throws InterruptedException { .expectComplete() .verify(DEFAULT_TIMEOUT); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); List spans = spanProcessor.getEndedSpans(); - List message = findSpans(spans, "EventHubs.message"); + List message = findSpans(spans, EVENT); assertMessageSpan(message.get(0), data); - List send = findSpans(spans, "EventHubs.send"); - assertSendSpan(send.get(0), Collections.singletonList(data), "EventHubs.send"); + List send = findSpans(spans, SEND); + assertSendSpan(send.get(0), Collections.singletonList(data)); - List received = findSpans(spans, "EventHubs.consume").stream() + List received = findSpans(spans, PROCESS).stream() .filter(s -> s == receivedSpan.get()).collect(toList()); - assertConsumerSpan(received.get(0), receivedMessage.get(), "EventHubs.consume"); + assertConsumerSpan(received.get(0), receivedMessage.get()); + assertNull(received.get(0).getAttribute(AttributeKey.stringKey("messaging.consumer.group.name"))); } @Test @@ -174,7 +173,7 @@ public void sendAndReceive() throws InterruptedException { AtomicReference receivedSpan = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(2); - spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || span.getName().equals("EventHubs.send")); + spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer .receive() .take(1) @@ -192,14 +191,14 @@ public void sendAndReceive() throws InterruptedException { List spans = spanProcessor.getEndedSpans(); - List message = findSpans(spans, "EventHubs.message"); + List message = findSpans(spans, EVENT); assertMessageSpan(message.get(0), data); - List send = findSpans(spans, "EventHubs.send"); - assertSendSpan(send.get(0), Collections.singletonList(data), "EventHubs.send"); + List send = findSpans(spans, SEND); + assertSendSpan(send.get(0), Collections.singletonList(data)); - List received = findSpans(spans, "EventHubs.consume").stream() + List received = findSpans(spans, PROCESS).stream() .filter(s -> s == receivedSpan.get()).collect(toList()); - assertConsumerSpan(received.get(0), receivedMessage.get(), "EventHubs.consume"); + assertConsumerSpan(received.get(0), receivedMessage.get()); } @Test @@ -217,7 +216,7 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { createClients(otel); CountDownLatch latch = new CountDownLatch(2); - customSpanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || span.getName().equals("EventHubs.send")); + customSpanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer.receive() .take(1) @@ -231,25 +230,25 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { .expectComplete() .verify(DEFAULT_TIMEOUT); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); List spans = customSpanProcessor.getEndedSpans(); - List message = findSpans(spans, "EventHubs.message"); + List message = findSpans(spans, EVENT); assertMessageSpan(message.get(0), data); - List send = findSpans(spans, "EventHubs.send"); - assertSendSpan(send.get(0), Collections.singletonList(data), "EventHubs.send"); + List send = findSpans(spans, SEND); + assertSendSpan(send.get(0), Collections.singletonList(data)); - List received = findSpans(spans, "EventHubs.consume").stream() + List received = findSpans(spans, PROCESS).stream() .filter(s -> s == receivedSpan.get()).collect(toList()); - assertConsumerSpan(received.get(0), receivedMessage.get(), "EventHubs.consume"); + assertConsumerSpan(received.get(0), receivedMessage.get()); } @Test public void sendAndReceiveParallel() throws InterruptedException { int messageCount = 5; CountDownLatch latch = new CountDownLatch(messageCount); - spanProcessor.notifyIfCondition(latch, span -> span.getName().equals("EventHubs.consume")); + spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS)); StepVerifier.create(consumer .receive() .take(messageCount) @@ -275,14 +274,14 @@ public void sendAndReceiveParallel() throws InterruptedException { assertTrue(latch.await(20, TimeUnit.SECONDS)); List spans = spanProcessor.getEndedSpans(); - List received = findSpans(spans, "EventHubs.consume"); + List received = findSpans(spans, PROCESS); assertTrue(messageCount <= received.size()); } @Test public void sendBuffered() throws InterruptedException { CountDownLatch latch = new CountDownLatch(3); - spanProcessor.notifyIfCondition(latch, span -> span.getName().equals("EventHubs.consume") || span.getName().equals("EventHubs.send")); + spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS) || hasOperationName(span, SEND)); EventHubBufferedProducerAsyncClient bufferedProducer = toClose(new EventHubBufferedProducerClientBuilder() .credential(TestUtils.getPipelineCredential(cachedCredential)) @@ -303,7 +302,7 @@ public void sendBuffered() throws InterruptedException { // Using a specific partition in the case that an epoch receiver was created // (i.e. EventHubConsumerAsyncClientIntegrationTest), which this scenario will fail when trying to create a // receiver. - SendOptions sendOptions = new SendOptions().setPartitionId("3"); + SendOptions sendOptions = new SendOptions().setPartitionId("0"); Boolean partitionIdExists = bufferedProducer.getPartitionIds() .any(id -> id.equals(sendOptions.getPartitionId())) .block(Duration.ofSeconds(30)); @@ -333,14 +332,15 @@ public void sendBuffered() throws InterruptedException { assertTrue(latch.await(10, TimeUnit.SECONDS)); List spans = spanProcessor.getEndedSpans(); - List message = findSpans(spans, "EventHubs.message"); + List message = findSpans(spans, EVENT); assertMessageSpan(message.get(0), event1); assertMessageSpan(message.get(1), event2); - List send = findSpans(spans, "EventHubs.send"); - assertSendSpan(send.get(0), Arrays.asList(event1, event2), "EventHubs.send"); + List send = findSpans(spans, SEND); + assertSendSpan(send.get(0), Arrays.asList(event1, event2)); + assertEquals(sendOptions.getPartitionId(), send.get(0).getAttribute(AttributeKey.stringKey("messaging.destination.partition.id"))); - List received = findSpans(spans, "EventHubs.consume"); + List received = findSpans(spans, PROCESS); assertEquals(2, received.size()); } @@ -361,10 +361,10 @@ public void syncReceive() { assertEquals(2, receivedMessages.size()); List spans = spanProcessor.getEndedSpans(); - assertEquals(0, findSpans(spans, "EventHubs.process").size()); + assertEquals(0, findSpans(spans, PROCESS).size()); - List received = findSpans(spans, "EventHubs.receiveFromPartition"); - assertSyncConsumerSpan(received.get(0), receivedMessages, "EventHubs.receiveFromPartition"); + List received = findSpans(spans, RECEIVE); + assertSyncConsumerSpan(received.get(0), receivedMessages); } @Test @@ -385,10 +385,10 @@ public void syncReceiveWithOptions() { assertEquals(2, receivedMessages.size()); List spans = spanProcessor.getEndedSpans(); - assertEquals(0, findSpans(spans, "EventHubs.process").size()); + assertEquals(0, findSpans(spans, PROCESS).size()); - List received = findSpans(spans, "EventHubs.receiveFromPartition"); - assertSyncConsumerSpan(received.get(0), receivedMessages, "EventHubs.receiveFromPartition"); + List received = findSpans(spans, RECEIVE); + assertSyncConsumerSpan(received.get(0), receivedMessages); } @Test @@ -398,10 +398,10 @@ public void syncReceiveTimeout() { .stream().collect(toList()); List spans = spanProcessor.getEndedSpans(); - assertEquals(0, findSpans(spans, "EventHubs.process").size()); + assertEquals(0, findSpans(spans, PROCESS).size()); - List received = findSpans(spans, "EventHubs.receiveFromPartition"); - assertSyncConsumerSpan(received.get(0), receivedMessages, "EventHubs.receiveFromPartition"); + List received = findSpans(spans, RECEIVE); + assertSyncConsumerSpan(received.get(0), receivedMessages); } @Test @@ -410,11 +410,12 @@ public void sendAndProcess() throws InterruptedException { AtomicReference receivedMessage = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(2); - spanProcessor.notifyIfCondition(latch, span -> span == currentInProcess.get() || span.getName().equals("EventHubs.send")); + spanProcessor.notifyIfCondition(latch, span -> span == currentInProcess.get() || hasOperationName(span, SEND)); StepVerifier.create(producer.send(data, new SendOptions().setPartitionId(PARTITION_ID))) .expectComplete() .verify(DEFAULT_TIMEOUT); + EventHubClientBuilder builder = createBuilder(); processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) @@ -438,21 +439,30 @@ public void sendAndProcess() throws InterruptedException { assertTrue(currentInProcess.get().getSpanContext().isValid()); List spans = spanProcessor.getEndedSpans(); - List message = findSpans(spans, "EventHubs.message"); + List message = findSpans(spans, EVENT); assertMessageSpan(message.get(0), data); - List send = findSpans(spans, "EventHubs.send"); - assertSendSpan(send.get(0), Collections.singletonList(data), "EventHubs.send"); + List send = findSpans(spans, SEND); + assertSendSpan(send.get(0), Collections.singletonList(data)); - List processed = findSpans(spans, "EventHubs.process") + List processed = findSpans(spans, PROCESS) .stream().filter(p -> p == currentInProcess.get()).collect(toList()); assertEquals(1, processed.size()); - assertConsumerSpan(processed.get(0), receivedMessage.get(), "EventHubs.process"); + assertConsumerSpan(processed.get(0), receivedMessage.get()); + assertNull(processed.get(0).getAttribute(AttributeKey.stringKey("messaging.consumer.group.name"))); + + SpanContext parentSpanContext = currentInProcess.get().getSpanContext(); + List checkpointed = findSpans(spans, CHECKPOINT) + .stream().filter(c -> c.getParentSpanContext().getSpanId() + .equals(parentSpanContext.getSpanId())).collect(toList()); + assertEquals(1, checkpointed.size()); + assertCheckpointSpan(checkpointed.get(0), parentSpanContext); } @Test @SuppressWarnings("try") public void sendNotInstrumentedAndProcess() throws InterruptedException { + EventHubProducerAsyncClient notInstrumentedProducer = toClose(createBuilder() .clientOptions(new ClientOptions().setTracingOptions(new TracingOptions().setEnabled(false))) .buildAsyncProducerClient()); @@ -461,17 +471,17 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { EventData message2 = new EventData(CONTENTS_BYTES); List received = new ArrayList<>(); CountDownLatch latch = new CountDownLatch(2); - spanProcessor.notifyIfCondition(latch, span -> span.getName().equals("EventHubs.process") && !span.getParentSpanContext().isValid()); + spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS) && !span.getParentSpanContext().isValid()); StepVerifier.create(notInstrumentedProducer.send(Arrays.asList(message1, message2), new SendOptions().setPartitionId(PARTITION_ID))) - .expectComplete() - .verify(DEFAULT_TIMEOUT); + .expectComplete() + .verify(DEFAULT_TIMEOUT); assertNull(message1.getProperties().get("traceparent")); assertNull(message2.getProperties().get("traceparent")); Span test = GlobalOpenTelemetry.getTracer("test") - .spanBuilder("test") - .startSpan(); + .spanBuilder("test") + .startSpan(); EventHubClientBuilder builder = createBuilder(); @@ -499,19 +509,25 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { List spans = spanProcessor.getEndedSpans(); - List processed = findSpans(spans, "EventHubs.process").stream() - .filter(s -> !s.getParentSpanContext().isValid()) - .collect(toList()); + List processed = findSpans(spans, PROCESS).stream() + .filter(s -> !s.getParentSpanContext().isValid()) + .collect(toList()); assertTrue(processed.size() >= 2); - assertConsumerSpan(processed.get(0), received.get(0), "EventHubs.process"); + assertConsumerSpan(processed.get(0), received.get(0)); + List checkpointed = findSpans(spans, CHECKPOINT).stream().collect(toList()); for (int i = 1; i < processed.size(); i++) { - assertConsumerSpan(processed.get(i), received.get(i), "EventHubs.process"); - assertNotEquals(processed.get(0).getSpanContext().getTraceId(), processed.get(i).getSpanContext().getTraceId()); + assertConsumerSpan(processed.get(i), received.get(i)); + SpanContext parentSpanContext = processed.get(i).getSpanContext(); + assertNotEquals(processed.get(0).getSpanContext().getTraceId(), parentSpanContext.getTraceId()); + List checkpointedChildren = checkpointed.stream() + .filter(c -> c.getParentSpanContext().getSpanId().equals(parentSpanContext.getSpanId())) + .collect(toList()); + assertEquals(1, checkpointedChildren.size()); + assertCheckpointSpan(checkpointedChildren.get(0), parentSpanContext); } } - @Test public void sendAndProcessBatch() throws InterruptedException { EventData message1 = new EventData(CONTENTS_BYTES); @@ -552,18 +568,25 @@ public void sendAndProcessBatch() throws InterruptedException { List spans = spanProcessor.getEndedSpans(); - List messages = findSpans(spans, "EventHubs.message"); + List messages = findSpans(spans, EVENT); assertMessageSpan(messages.get(0), message1); assertMessageSpan(messages.get(1), message2); - List send = findSpans(spans, "EventHubs.send"); + List send = findSpans(spans, SEND); - assertSendSpan(send.get(0), Arrays.asList(message1, message2), "EventHubs.send"); + assertSendSpan(send.get(0), Arrays.asList(message1, message2)); - List processed = findSpans(spans, "EventHubs.process") + List processed = findSpans(spans, PROCESS) .stream().filter(p -> p == currentInProcess.get()).collect(toList()); assertEquals(1, processed.size()); - assertConsumerSpan(processed.get(0), received.get(), "EventHubs.process", StatusCode.UNSET); + assertConsumerSpan(processed.get(0), received.get(), StatusCode.UNSET); + + SpanContext parentSpanContext = currentInProcess.get().getSpanContext(); + List checkpointed = findSpans(spans, CHECKPOINT) + .stream().filter(c -> c.getParentSpanContext().getSpanId() + .equals(parentSpanContext.getSpanId())).collect(toList()); + assertEquals(1, checkpointed.size()); + assertCheckpointSpan(checkpointed.get(0), parentSpanContext); } @Test @@ -571,7 +594,7 @@ public void sendProcessAndFail() throws InterruptedException { AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference> received = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(2); - spanProcessor.notifyIfCondition(latch, span -> span == currentInProcess.get() || span.getName().equals("EventHubs.send")); + spanProcessor.notifyIfCondition(latch, span -> span == currentInProcess.get() || hasOperationName(span, SEND)); StepVerifier.create(producer.send(data, new SendOptions().setPartitionId(PARTITION_ID))) .expectComplete() @@ -600,28 +623,36 @@ public void sendProcessAndFail() throws InterruptedException { processor.stop(); List spans = spanProcessor.getEndedSpans(); - List processed = findSpans(spans, "EventHubs.process") + List processed = findSpans(spans, PROCESS) .stream().filter(p -> p == currentInProcess.get()) .collect(toList()); assertEquals(1, processed.size()); - assertConsumerSpan(processed.get(0), received.get(), "EventHubs.process", StatusCode.ERROR); + assertConsumerSpan(processed.get(0), received.get(), StatusCode.ERROR); + assertEquals(RuntimeException.class.getName(), processed.get(0).getAttribute(AttributeKey.stringKey("error.type"))); + + SpanContext parentSpanContext = currentInProcess.get().getSpanContext(); + List checkpointed = findSpans(spans, CHECKPOINT) + .stream().filter(c -> c.getParentSpanContext().getSpanId() + .equals(parentSpanContext.getSpanId())).collect(toList()); + assertEquals(1, checkpointed.size()); + assertCheckpointSpan(checkpointed.get(0), parentSpanContext); } private void assertMessageSpan(ReadableSpan actual, EventData message) { - assertEquals("EventHubs.message", actual.getName()); assertEquals(SpanKind.PRODUCER, actual.getKind()); assertEquals(StatusCode.UNSET, actual.toSpanData().getStatus().getStatusCode()); - assertNull(actual.getAttribute(AttributeKey.stringKey("messaging.operation"))); + assertEquals("event", actual.getAttribute(OPERATION_NAME_ATTRIBUTE)); + assertNull(actual.getAttribute(OPERATION_TYPE_ATTRIBUTE)); String traceparent = "00-" + actual.getSpanContext().getTraceId() + "-" + actual.getSpanContext().getSpanId() + "-01"; assertEquals(message.getProperties().get("Diagnostic-Id"), traceparent); assertEquals(message.getProperties().get("traceparent"), traceparent); } - private void assertSendSpan(ReadableSpan actual, List messages, String spanName) { - assertEquals(spanName, actual.getName()); + private void assertSendSpan(ReadableSpan actual, List messages) { assertEquals(SpanKind.CLIENT, actual.getKind()); assertEquals(StatusCode.UNSET, actual.toSpanData().getStatus().getStatusCode()); - assertEquals("publish", actual.getAttribute(AttributeKey.stringKey("messaging.operation"))); + assertEquals("send", actual.getAttribute(OPERATION_NAME_ATTRIBUTE)); + assertEquals("publish", actual.getAttribute(OPERATION_TYPE_ATTRIBUTE)); if (messages.size() > 1) { assertEquals(messages.size(), actual.getAttribute(AttributeKey.longKey("messaging.batch.message_count"))); } @@ -635,12 +666,12 @@ private void assertSendSpan(ReadableSpan actual, List messages, Strin } } - private void assertSyncConsumerSpan(ReadableSpan actual, List messages, String spanName) { - assertEquals(spanName, actual.getName()); - assertEquals(SpanKind.CLIENT, actual.getKind()); + private void assertSyncConsumerSpan(ReadableSpan actual, List messages) { + assertEquals(SpanKind.CONSUMER, actual.getKind()); assertEquals(StatusCode.UNSET, actual.toSpanData().getStatus().getStatusCode()); List links = actual.toSpanData().getLinks(); - assertEquals("receive", actual.getAttribute(AttributeKey.stringKey("messaging.operation"))); + assertEquals("receive", actual.getAttribute(OPERATION_NAME_ATTRIBUTE)); + assertEquals("receive", actual.getAttribute(OPERATION_TYPE_ATTRIBUTE)); if (messages.size() > 1) { assertEquals(messages.size(), actual.getAttribute(AttributeKey.longKey("messaging.batch.message_count"))); } @@ -654,33 +685,40 @@ private void assertSyncConsumerSpan(ReadableSpan actual, List me } } - private void assertConsumerSpan(ReadableSpan actual, EventData message, String spanName) { - assertEquals(spanName, actual.getName()); + private void assertConsumerSpan(ReadableSpan actual, EventData message) { + SpanData spanData = actual.toSpanData(); assertEquals(SpanKind.CONSUMER, actual.getKind()); - assertEquals(StatusCode.UNSET, actual.toSpanData().getStatus().getStatusCode()); - assertEquals(0, actual.toSpanData().getLinks().size()); - assertEquals("process", actual.getAttribute(AttributeKey.stringKey("messaging.operation"))); + assertEquals(StatusCode.UNSET, spanData.getStatus().getStatusCode()); + assertNotNull(actual.getAttribute(OPERATION_NAME_ATTRIBUTE)); + assertNotNull(actual.getAttribute(OPERATION_TYPE_ATTRIBUTE)); + assertEquals(PARTITION_ID, actual.getAttribute(AttributeKey.stringKey("messaging.destination.partition.id"))); String messageTraceparent = (String) message.getProperties().get("traceparent"); if (messageTraceparent == null) { + assertEquals(0, spanData.getLinks().size()); assertFalse(actual.getParentSpanContext().isValid()); } else { + assertEquals(1, spanData.getLinks().size()); + LinkData link = spanData.getLinks().get(0); + assertEquals(actual.getSpanContext().getTraceId(), link.getSpanContext().getTraceId()); + assertEquals(actual.getParentSpanContext().getSpanId(), link.getSpanContext().getSpanId()); String parent = "00-" + actual.getSpanContext().getTraceId() + "-" + actual.getParentSpanContext().getSpanId() + "-01"; assertEquals(messageTraceparent, parent); } } - private void assertConsumerSpan(ReadableSpan actual, List messages, String spanName, StatusCode status) { + private void assertConsumerSpan(ReadableSpan actual, List messages, StatusCode status) { logger.atInfo() .addKeyValue("linkCount", actual.toSpanData().getLinks().size()) .addKeyValue("receivedCount", messages.size()) .addKeyValue("batchSize", actual.getAttribute(AttributeKey.longKey("messaging.batch.message_count"))) .log("assertConsumerSpan"); - assertEquals(spanName, actual.getName()); assertEquals(SpanKind.CONSUMER, actual.getKind()); assertEquals(status, actual.toSpanData().getStatus().getStatusCode()); - assertEquals("process", actual.getAttribute(AttributeKey.stringKey("messaging.operation"))); + assertEquals("process", actual.getAttribute(OPERATION_NAME_ATTRIBUTE)); + assertEquals("process", actual.getAttribute(OPERATION_TYPE_ATTRIBUTE)); + assertNotNull(actual.getAttribute(AttributeKey.stringKey("messaging.destination.partition.id"))); List receivedMessagesWithTraceContext = messages.stream().filter(m -> m.getProperties().containsKey("traceparent")).collect(toList()); assertEquals(receivedMessagesWithTraceContext.size(), actual.toSpanData().getLinks().size()); @@ -700,99 +738,26 @@ private void assertConsumerSpan(ReadableSpan actual, List messages, S } } - private List findSpans(List spans, String spanName) { + private void assertCheckpointSpan(ReadableSpan actual, SpanContext parent) { + assertEquals(SpanKind.INTERNAL, actual.getKind()); + assertEquals(StatusCode.UNSET, actual.toSpanData().getStatus().getStatusCode()); + assertEquals("checkpoint", actual.getAttribute(OPERATION_NAME_ATTRIBUTE)); + assertEquals("settle", actual.getAttribute(OPERATION_TYPE_ATTRIBUTE)); + assertEquals(parent.getTraceId(), actual.getSpanContext().getTraceId()); + assertEquals(parent.getSpanId(), actual.getParentSpanContext().getSpanId()); + + assertNull(actual.getAttribute(AttributeKey.stringKey("messaging.consumer.group.name"))); + assertNotNull(actual.getAttribute(AttributeKey.stringKey("messaging.destination.partition.id"))); + } + + private List findSpans(List spans, OperationName operationName) { + String spanName = TestUtils.getSpanName(operationName, getEventHubName()); return spans.stream() .filter(s -> s.getName().equals(spanName)) .collect(toList()); } - static class TestSpanProcessor implements SpanProcessor { - private static final ClientLogger LOGGER = new ClientLogger(TestSpanProcessor.class); - private final ConcurrentLinkedDeque spans = new ConcurrentLinkedDeque<>(); - private final String entityName; - private final String namespace; - private final String testName; - - private final AtomicReference> notifier = new AtomicReference<>(); - - TestSpanProcessor(String namespace, String entityName, String testName) { - this.namespace = namespace; - this.entityName = entityName; - this.testName = testName; - } - public List getEndedSpans() { - return new ArrayList<>(spans); - } - - @Override - public void onStart(Context context, ReadWriteSpan readWriteSpan) { - } - - @Override - public boolean isStartRequired() { - return false; - } - - @Override - public void onEnd(ReadableSpan readableSpan) { - SpanData span = readableSpan.toSpanData(); - - InstrumentationScopeInfo instrumentationScopeInfo = span.getInstrumentationScopeInfo(); - LoggingEventBuilder log = LOGGER.atInfo() - .addKeyValue("testName", testName) - .addKeyValue("name", span.getName()) - .addKeyValue("traceId", span.getTraceId()) - .addKeyValue("spanId", span.getSpanId()) - .addKeyValue("parentSpanId", span.getParentSpanId()) - .addKeyValue("kind", span.getKind()) - .addKeyValue("tracerName", instrumentationScopeInfo.getName()) - .addKeyValue("tracerVersion", instrumentationScopeInfo.getVersion()) - .addKeyValue("attributes", span.getAttributes()); - - for (int i = 0; i < span.getLinks().size(); i++) { - LinkData link = span.getLinks().get(i); - log.addKeyValue("linkTraceId" + i, link.getSpanContext().getTraceId()) - .addKeyValue("linkSpanId" + i, link.getSpanContext().getSpanId()) - .addKeyValue("linkAttributes" + i, link.getAttributes()); - } - log.log("got span"); - - spans.add(readableSpan); - Consumer filter = notifier.get(); - if (filter != null) { - filter.accept(readableSpan); - } - - // Various attribute keys can be found in: - // sdk/core/azure-core-metrics-opentelemetry/src/main/java/com/azure/core/metrics/opentelemetry/OpenTelemetryAttributes.java - // sdk/core/azure-core-tracing-opentelemetry/src/main/java/com/azure/core/tracing/opentelemetry/OpenTelemetryUtils.java - assertEquals("Microsoft.EventHub", readableSpan.getAttribute(AttributeKey.stringKey("az.namespace"))); - assertEquals("eventhubs", readableSpan.getAttribute(AttributeKey.stringKey("messaging.system"))); - assertEquals(entityName, readableSpan.getAttribute(AttributeKey.stringKey("messaging.destination.name"))); - assertEquals(namespace, readableSpan.getAttribute(AttributeKey.stringKey("net.peer.name"))); - } - - public void notifyIfCondition(CountDownLatch countDownLatch, Predicate filter) { - notifier.set((span) -> { - if (filter.test(span)) { - LOGGER.atInfo() - .addKeyValue("traceId", span.getSpanContext().getTraceId()) - .addKeyValue("spanId", span.getSpanContext().getSpanId()) - .log("condition met"); - countDownLatch.countDown(); - } - }); - } - - @Override - public boolean isEndRequired() { - return true; - } - - @Override - public CompletableResultCode shutdown() { - notifier.set(null); - return CompletableResultCode.ofSuccess(); - } + private boolean hasOperationName(ReadableSpan span, OperationName operationName) { + return operationName.toString().equals(span.getAttribute(OPERATION_NAME_ATTRIBUTE)); } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentationTests.java new file mode 100644 index 0000000000000..b0c2d7049f814 --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentationTests.java @@ -0,0 +1,649 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs.implementation.instrumentation; + +import com.azure.core.amqp.AmqpMessageConstant; +import com.azure.core.amqp.exception.AmqpErrorCondition; +import com.azure.core.amqp.exception.AmqpException; +import com.azure.core.amqp.models.AmqpAnnotatedMessage; +import com.azure.core.amqp.models.AmqpMessageBody; +import com.azure.core.test.utils.metrics.TestCounter; +import com.azure.core.test.utils.metrics.TestHistogram; +import com.azure.core.test.utils.metrics.TestMeasurement; +import com.azure.core.test.utils.metrics.TestMeter; +import com.azure.core.tracing.opentelemetry.OpenTelemetryTracingOptions; +import com.azure.core.util.TracingOptions; +import com.azure.core.util.tracing.Tracer; +import com.azure.core.util.tracing.TracerProvider; +import com.azure.messaging.eventhubs.CheckpointStore; +import com.azure.messaging.eventhubs.EventData; +import com.azure.messaging.eventhubs.SampleCheckpointStore; +import com.azure.messaging.eventhubs.TestSpanProcessor; +import com.azure.messaging.eventhubs.TestUtils; +import com.azure.messaging.eventhubs.models.Checkpoint; +import com.azure.messaging.eventhubs.models.EventBatchContext; +import com.azure.messaging.eventhubs.models.EventContext; +import com.azure.messaging.eventhubs.models.PartitionContext; +import com.azure.messaging.eventhubs.models.PartitionEvent; +import com.azure.messaging.eventhubs.models.PartitionOwnership; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.data.SpanData; +import org.apache.qpid.proton.amqp.Symbol; +import org.apache.qpid.proton.amqp.messaging.ApplicationProperties; +import org.apache.qpid.proton.amqp.messaging.MessageAnnotations; +import org.apache.qpid.proton.message.Message; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.Exceptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; +import reactor.util.context.ContextView; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.azure.core.amqp.AmqpMessageConstant.ENQUEUED_TIME_UTC_ANNOTATION_NAME; +import static com.azure.messaging.eventhubs.TestUtils.assertAllAttributes; +import static com.azure.messaging.eventhubs.TestUtils.assertSpanStatus; +import static com.azure.messaging.eventhubs.TestUtils.attributesToMap; +import static com.azure.messaging.eventhubs.TestUtils.getSpanName; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.PROCESS; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.RECEIVE; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.CHECKPOINT; +import static io.opentelemetry.api.trace.SpanKind.CONSUMER; +import static io.opentelemetry.api.trace.SpanKind.INTERNAL; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class EventHubsConsumerInstrumentationTests { + private static final String FQDN = "fqdn"; + private static final String ENTITY_NAME = "entityName"; + private static final String CONSUMER_GROUP = "consumerGroup"; + private static final String TRACEPARENT1 = "00-1123456789abcdef0123456789abcdef-0123456789abcdef-01"; + private static final String TRACEID1 = TRACEPARENT1.substring(3, 35); + private static final String SPANID1 = TRACEPARENT1.substring(36, 52); + private static final String TRACEPARENT2 = "00-2123456789abcdef0123456789abcdef-0123456789abcdef-01"; + private static final String TRACEPARENT3 = "00-3123456789abcdef0123456789abcdef-0123456789abcdef-01"; + private Tracer tracer; + private TestMeter meter; + private TestSpanProcessor spanProcessor; + + private CheckpointStore checkpointStore; + @BeforeEach + public void setup(TestInfo testInfo) { + spanProcessor = new TestSpanProcessor(FQDN, ENTITY_NAME, testInfo.getDisplayName()); + OpenTelemetry otel = OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .addSpanProcessor(spanProcessor) + .build()) + .build(); + + TracingOptions tracingOptions = new OpenTelemetryTracingOptions().setOpenTelemetry(otel); + tracer = TracerProvider.getDefaultProvider() + .createTracer("test", null, "Microsoft.EventHub", tracingOptions); + meter = new TestMeter(); + checkpointStore = new SampleCheckpointStore(); + } + + @AfterEach + public void teardown() { + spanProcessor.shutdown(); + spanProcessor.close(); + meter.close(); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SuppressWarnings("try") + public void startAsyncConsumeDisabledInstrumentation(boolean sync) { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(null, null, + FQDN, ENTITY_NAME, CONSUMER_GROUP, sync); + + try (InstrumentationScope scope = + instrumentation.startAsyncConsume(createMessage(Instant.now()), "0")) { + assertNull(scope.getStartTime()); + } + + TestHistogram lag = meter.getHistograms().get("messaging.eventhubs.consumer.lag"); + assertNull(lag); + assertEquals(0, spanProcessor.getEndedSpans().size()); + } + + @Test + @SuppressWarnings("try") + public void startAsyncConsumeSyncReportsLag() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, true); + + int measurements = 3; + Integer[] lags = new Integer[]{ -10, 0, 10}; + Integer[] expectedLags = new Integer[]{ 10, 0, 0 }; + String[] partitionIds = new String[]{"1", "2", "3"}; + + for (int i = 0; i < measurements; i++) { + try (InstrumentationScope scope = + instrumentation.startAsyncConsume(createMessage(Instant.now().plusSeconds(lags[i])), partitionIds[i])) { + // lag is reported about received message and don't need to report processing errors + // so those will be ignored + if (i == 0) { + scope.setCancelled(); + } else if (i == 1) { + scope.setError(new RuntimeException("error")); + } + } + } + + TestHistogram lag = meter.getHistograms().get("messaging.eventhubs.consumer.lag"); + assertNotNull(lag); + assertEquals(measurements, lag.getMeasurements().size()); + for (int i = 0; i < measurements; i++) { + assertEquals(expectedLags[i], lag.getMeasurements().get(i).getValue(), 10); + assertAllAttributes(FQDN, ENTITY_NAME, partitionIds[i], CONSUMER_GROUP, null, null, + lag.getMeasurements().get(i).getAttributes()); + } + + // sync consumer reports spans in different instrumentation point + assertEquals(0, spanProcessor.getEndedSpans().size()); + + assertEquals(0, meter.getHistograms().get("messaging.process.duration").getMeasurements().size()); + } + + @Test + @SuppressWarnings("try") + public void startAsyncConsumeAsyncReportsLagAndSpans() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + int measurements = 3; + Integer[] lags = new Integer[]{ -10, 0, 10}; + String[] partitionIds = new String[]{"1", "2", "3"}; + String[] expectedErrors = new String[3]; + for (int i = 0; i < measurements; i++) { + try (InstrumentationScope scope = + instrumentation.startAsyncConsume(createMessage(Instant.now().plusSeconds(lags[i])), partitionIds[i])) { + // lag is reported about received message and don't need to report processing errors + // so those will be ignored on lag, but will be reflected on processing spans + if (i == 0) { + expectedErrors[i] = "cancelled"; + scope.setCancelled(); + } else if (i == 1) { + expectedErrors[i] = RuntimeException.class.getName(); + scope.setError(new RuntimeException("test")); + } + + assertTrue(scope.getStartTime().toEpochMilli() <= Instant.now().toEpochMilli()); + assertTrue(Span.current().getSpanContext().isValid()); + } + } + + assertEquals(measurements, spanProcessor.getEndedSpans().size()); + for (int i = 0; i < measurements; i++) { + SpanData span = spanProcessor.getEndedSpans().get(i).toSpanData(); + assertEquals(getSpanName(PROCESS, ENTITY_NAME), span.getName()); + Map attributes = attributesToMap(span.getAttributes()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionIds[i], CONSUMER_GROUP, expectedErrors[i], + PROCESS, attributes); + assertNotNull(attributes.get("messaging.eventhubs.message.enqueued_time")); + assertSpanStatus(i == 0 ? "cancelled" : i == 1 ? "test" : null, span); + assertProcessDuration(null, partitionIds[i], expectedErrors[i]); + } + } + + @Test + @SuppressWarnings("try") + public void asyncConsumerSpansHaveLinks() throws InterruptedException { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + Instant enqueuedTime = Instant.now(); + Message message = createMessage(enqueuedTime, TRACEPARENT1); + int durationMillis = 100; + try (InstrumentationScope scope = instrumentation.startAsyncConsume(message, "0")) { + Thread.sleep(100); + assertEquals(TRACEID1, Span.current().getSpanContext().getTraceId()); + } + + // async consumer should report spans + assertEquals(1, spanProcessor.getEndedSpans().size()); + SpanData span = spanProcessor.getEndedSpans().get(0).toSpanData(); + assertEquals(getSpanName(PROCESS, ENTITY_NAME), span.getName()); + assertEquals(CONSUMER, span.getKind()); + assertEquals(SPANID1, span.getParentSpanId()); + assertTrue(span.getEndEpochNanos() - span.getStartEpochNanos() >= durationMillis * 1_000_000); + + Map attributes = attributesToMap(span.getAttributes()); + assertAllAttributes(FQDN, ENTITY_NAME, "0", CONSUMER_GROUP, null, PROCESS, attributes); + assertEquals(enqueuedTime.getEpochSecond(), attributes.get("messaging.eventhubs.message.enqueued_time")); + + assertTrue(span.getLinks().get(0).getSpanContext().isValid()); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SuppressWarnings("try") + public void syncReceiveDisabledInstrumentation(boolean sync) { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(null, null, + FQDN, ENTITY_NAME, CONSUMER_GROUP, sync); + + Flux events = Flux.just(createPartitionEvent(Instant.now(), null, "0")); + instrumentation.syncReceive(events, "0"); + + TestHistogram lag = meter.getHistograms().get("messaging.eventhubs.consumer.lag"); + assertNull(lag); + assertEquals(0, spanProcessor.getEndedSpans().size()); + } + + public static Stream syncReceiveErrors() { + return Stream.of( + Arguments.of(false, null, null, null), + Arguments.of(true, null, "cancelled", "cancelled"), + Arguments.of(false, new RuntimeException("test"), RuntimeException.class.getName(), "test"), + Arguments.of(false, Exceptions.propagate(new RuntimeException("test")), RuntimeException.class.getName(), "test") + ); + } + + @ParameterizedTest + @MethodSource("syncReceiveErrors") + @SuppressWarnings("try") + public void syncReceiveOneEvent(boolean cancel, Throwable error, String expectedErrorType, String spanDescription) { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + String partitionId = "0"; + Flux events = Flux.just(createPartitionEvent(Instant.now(), null, partitionId)) + .flatMap(e -> error == null ? Mono.just(e) : Mono.error(error)); + + StepVerifier.Step stepVerifier = StepVerifier.create(instrumentation.syncReceive(events, partitionId)); + + if (cancel) { + stepVerifier.thenCancel().verify(); + } else if (error != null) { + stepVerifier.expectErrorMessage(error.getMessage()).verify(); + } else { + stepVerifier + .expectNextCount(1) + .expectComplete() + .verify(); + } + + assertOperationDuration(RECEIVE, partitionId, expectedErrorType); + assertConsumedCount(expectedErrorType == null ? 1 : 0, partitionId, null, RECEIVE); + assertReceiveSpan(expectedErrorType == null ? 1 : 0, partitionId, expectedErrorType, spanDescription); + } + + @ParameterizedTest + @MethodSource("syncReceiveErrors") + @SuppressWarnings("try") + public void syncReceiveBatchEvent(boolean cancel, Throwable error, String expectedErrorType, String spanDescription) { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + String partitionId = "1"; + + int count = 3; + Flux events = Flux.just( + createPartitionEvent(Instant.now(), TRACEPARENT1, partitionId), + createPartitionEvent(Instant.now(), TRACEPARENT2, partitionId), + createPartitionEvent(Instant.now(), TRACEPARENT3, partitionId)); + + if (error != null) { + events = events.concatWith(Mono.error(error)); + } + + StepVerifier.Step stepVerifier = StepVerifier.create(instrumentation.syncReceive(events, partitionId)) + .expectNextCount(count); + + if (cancel) { + stepVerifier + .thenCancel().verify(); + } else if (error != null) { + stepVerifier + .expectErrorMessage(error.getMessage()) + .verify(); + } else { + stepVerifier + .expectComplete() + .verify(); + } + + assertOperationDuration(RECEIVE, partitionId, expectedErrorType); + assertConsumedCount(count, partitionId, null, RECEIVE); + SpanData span = assertReceiveSpan(count, partitionId, expectedErrorType, spanDescription); + assertEquals(count, span.getLinks().size()); + for (int j = 0; j < count; j++) { + LinkData link = span.getLinks().get(j); + assertTrue(link.getSpanContext().isValid()); + assertNotNull(link.getAttributes().get(AttributeKey.longKey("messaging.eventhubs.message.enqueued_time"))); + } + } + + public static Stream processErrors() { + AmqpException amqpException = new AmqpException(false, AmqpErrorCondition.SERVER_BUSY_ERROR, null, new RuntimeException("test"), null); + return Stream.of( + Arguments.of(null, null, null), + Arguments.of(new RuntimeException("test"), RuntimeException.class.getName(), "test"), + Arguments.of(amqpException, amqpException.getErrorCondition().getErrorCondition(), "test"), + Arguments.of(Exceptions.propagate(new RuntimeException("test")), RuntimeException.class.getName(), "test") + ); + } + + @ParameterizedTest + @MethodSource("processErrors") + @SuppressWarnings("try") + public void processOneEvent(Throwable error, String expectedErrorType, String spanDescription) throws InterruptedException { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + String partitionId = "0"; + try (InstrumentationScope scope = instrumentation.startProcess(createEventContext(Instant.now(), TRACEPARENT1, partitionId))) { + scope.setError(error); + Thread.sleep(200); + } + + assertProcessDuration(Duration.ofMillis(200), partitionId, expectedErrorType); + assertConsumedCount(1, partitionId, expectedErrorType, PROCESS); + SpanData span = assertProcessSpan(partitionId, expectedErrorType, spanDescription); + assertNull(span.getAttributes().get(AttributeKey.longKey("messaging.batch.message_count"))); + assertNotNull(span.getAttributes().get(AttributeKey.longKey("messaging.eventhubs.message.enqueued_time"))); + + assertEquals(1, span.getLinks().size()); + LinkData link = span.getLinks().get(0); + assertTrue(link.getSpanContext().isValid()); + } + + @ParameterizedTest + @MethodSource("processErrors") + @SuppressWarnings("try") + public void processBatch(Throwable error, String expectedErrorType, String spanDescription) throws InterruptedException { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + String partitionId = "1"; + + int count = 3; + List events = Arrays.asList( + createEventData(Instant.now(), TRACEPARENT1), + createEventData(Instant.now(), TRACEPARENT2), + createEventData(Instant.now(), TRACEPARENT3)); + PartitionContext partitionContext = new PartitionContext(FQDN, ENTITY_NAME, CONSUMER_GROUP, partitionId); + EventBatchContext batchContext = new EventBatchContext(partitionContext, events, checkpointStore, null); + + try (InstrumentationScope scope = instrumentation.startProcess(batchContext)) { + scope.setError(error); + Thread.sleep(200); + } + + assertProcessDuration(Duration.ofMillis(200), partitionId, expectedErrorType); + assertConsumedCount(3, partitionId, expectedErrorType, PROCESS); + SpanData span = assertProcessSpan(partitionId, expectedErrorType, spanDescription); + assertEquals(count, span.getAttributes().get(AttributeKey.longKey("messaging.batch.message_count"))); + assertNull(span.getAttributes().get(AttributeKey.longKey("messaging.eventhubs.message.enqueued_time"))); + + assertEquals(count, span.getLinks().size()); + for (int j = 0; j < count; j++) { + LinkData link = span.getLinks().get(j); + assertTrue(link.getSpanContext().isValid()); + assertNotNull(link.getAttributes().get(AttributeKey.longKey("messaging.eventhubs.message.enqueued_time"))); + } + } + + public static Stream checkpointErrors() { + return Stream.of( + Arguments.of(false, null, null, null), + Arguments.of(true, null, "cancelled", "cancelled"), + Arguments.of(false, new RuntimeException("test"), RuntimeException.class.getName(), "test"), + Arguments.of(false, Exceptions.propagate(new RuntimeException("test")), RuntimeException.class.getName(), "test") + ); + } + + @Test + public void checkpointWithDisabledInstrumentation() { + CheckpointStore inner = new SampleCheckpointStore(); + + EventHubsConsumerInstrumentation disabled = new EventHubsConsumerInstrumentation(null, null, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + CheckpointStore instrumented = InstrumentedCheckpointStore.create(inner, disabled); + assertSame(instrumented, inner); + } + + @ParameterizedTest + @MethodSource("checkpointErrors") + @SuppressWarnings("try") + public void checkpoint(boolean cancel, Throwable error, String expectedErrorType, String spanDescription) { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + String partitionId = "0"; + + CheckpointStore store = InstrumentedCheckpointStore.create( + new TestCheckpointStore(i -> error == null ? i : Mono.error(error)), + instrumentation); + + StepVerifier.FirstStep stepVerifier = + StepVerifier.create(store.updateCheckpoint(createCheckpoint(partitionId))); + + if (cancel) { + stepVerifier.thenCancel().verify(); + } else if (error != null) { + stepVerifier.expectErrorMessage(error.getMessage()).verify(); + } else { + stepVerifier.expectComplete().verify(); + } + + assertOperationDuration(CHECKPOINT, partitionId, expectedErrorType); + assertCheckpointSpan(partitionId, expectedErrorType, spanDescription); + } + + @Test + @SuppressWarnings("try") + public void checkpointPassesContextToDownstream() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + String partitionId = "0"; + + AtomicReference capturedContext = new AtomicReference<>(); + + CheckpointStore store = InstrumentedCheckpointStore.create( + new TestCheckpointStore(inner -> Mono.deferContextual(ctx -> { + capturedContext.set(ctx); + return inner; + })), + instrumentation); + + StepVerifier.create(store.updateCheckpoint(createCheckpoint(partitionId)) + .contextWrite(ctx -> ctx.put("foo", "bar"))) + .expectComplete() + .verify(); + + // testing internal details - we should have trace-context key with otel span in the context + // and also client-method-call-flag set by azure-core OtelTracer used to suppress nested spans + assertTrue((Boolean) capturedContext.get().get("client-method-call-flag")); + assertInstanceOf(io.opentelemetry.context.Context.class, capturedContext.get().get("trace-context")); + assertEquals("bar", capturedContext.get().get("foo")); + + assertOperationDuration(CHECKPOINT, partitionId, null); + assertCheckpointSpan(partitionId, null, null); + } + + private static Checkpoint createCheckpoint(String partitionId) { + return new Checkpoint() + .setFullyQualifiedNamespace(FQDN) + .setEventHubName(ENTITY_NAME) + .setSequenceNumber(1L) + .setPartitionId(partitionId) + .setOffset(2L) + .setConsumerGroup(CONSUMER_GROUP); + } + private static Message createMessage(Instant enqueuedTime) { + Message message = Message.Factory.create(); + message.setMessageAnnotations(new MessageAnnotations(Collections.singletonMap(Symbol.getSymbol(ENQUEUED_TIME_UTC_ANNOTATION_NAME.getValue()), enqueuedTime))); + message.setApplicationProperties(new ApplicationProperties(Collections.emptyMap())); + return message; + } + + private static Message createMessage(Instant enqueuedTime, String traceparent) { + Message message = Message.Factory.create(); + message.setMessageAnnotations(new MessageAnnotations(Collections.singletonMap(Symbol.getSymbol(ENQUEUED_TIME_UTC_ANNOTATION_NAME.getValue()), enqueuedTime))); + message.setApplicationProperties(new ApplicationProperties(Collections.singletonMap("traceparent", traceparent))); + return message; + } + + private static PartitionEvent createPartitionEvent(Instant enqueuedTime, String traceparent, String partitionId) { + PartitionContext context = new PartitionContext(FQDN, ENTITY_NAME, CONSUMER_GROUP, partitionId); + + EventData data = createEventData(enqueuedTime, traceparent); + return new PartitionEvent(context, data, null); + } + + private EventContext createEventContext(Instant enqueuedTime, String traceparent, String partitionId) { + PartitionContext context = new PartitionContext(FQDN, ENTITY_NAME, CONSUMER_GROUP, partitionId); + + EventData data = createEventData(enqueuedTime, traceparent); + return new EventContext(context, data, checkpointStore, null); + } + + private static EventData createEventData(Instant enqueuedTime, String traceparent) { + AmqpAnnotatedMessage annotatedMessage = new AmqpAnnotatedMessage( + AmqpMessageBody.fromData("foo".getBytes(StandardCharsets.UTF_8))); + annotatedMessage.getApplicationProperties().put("traceparent", traceparent); + annotatedMessage.getMessageAnnotations().put(AmqpMessageConstant.ENQUEUED_TIME_UTC_ANNOTATION_NAME.getValue(), enqueuedTime); + return TestUtils.createEventData(annotatedMessage, 25L, 14L, enqueuedTime); + } + + private SpanData assertReceiveSpan(int expectedBatchSize, String partitionId, String expectedErrorType, String spanDescription) { + assertEquals(1, spanProcessor.getEndedSpans().size()); + SpanData span = spanProcessor.getEndedSpans().get(0).toSpanData(); + assertEquals(getSpanName(RECEIVE, ENTITY_NAME), span.getName()); + assertEquals(CONSUMER, span.getKind()); + Map attributes = attributesToMap(span.getAttributes()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, RECEIVE, attributes); + assertSpanStatus(spanDescription, span); + + assertEquals((long) expectedBatchSize, attributes.get("messaging.batch.message_count")); + return span; + } + + private SpanData assertProcessSpan(String partitionId, String expectedErrorType, String spanDescription) { + assertEquals(1, spanProcessor.getEndedSpans().size()); + SpanData span = spanProcessor.getEndedSpans().get(0).toSpanData(); + assertEquals(getSpanName(PROCESS, ENTITY_NAME), span.getName()); + assertEquals(CONSUMER, span.getKind()); + Map attributes = attributesToMap(span.getAttributes()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, PROCESS, attributes); + assertSpanStatus(spanDescription, span); + return span; + } + + private SpanData assertCheckpointSpan(String partitionId, String expectedErrorType, String spanDescription) { + assertEquals(1, spanProcessor.getEndedSpans().size()); + SpanData span = spanProcessor.getEndedSpans().get(0).toSpanData(); + assertEquals(getSpanName(CHECKPOINT, ENTITY_NAME), span.getName()); + assertEquals(INTERNAL, span.getKind()); + Map attributes = attributesToMap(span.getAttributes()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, CHECKPOINT, attributes); + assertSpanStatus(spanDescription, span); + return span; + } + + private void assertOperationDuration(OperationName operationName, String partitionId, String expectedErrorType) { + TestHistogram operationDuration = meter.getHistograms().get("messaging.client.operation.duration"); + assertNotNull(operationDuration); + assertEquals(1, operationDuration.getMeasurements().size()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, + operationName, operationDuration.getMeasurements().get(0).getAttributes()); + } + + private void assertProcessDuration(Duration duration, String partitionId, String expectedErrorType) { + TestHistogram processDuration = meter.getHistograms().get("messaging.process.duration"); + assertNotNull(processDuration); + List> durationPerPartition = processDuration.getMeasurements().stream() + .filter(m -> partitionId.equals(m.getAttributes().get("messaging.destination.partition.id"))) + .collect(Collectors.toList()); + assertEquals(1, durationPerPartition.size()); + if (duration != null) { + double sec = getDoubleSeconds(duration); + assertEquals(sec, durationPerPartition.get(0).getValue(), sec); + } + + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, + PROCESS, durationPerPartition.get(0).getAttributes()); + } + + private void assertConsumedCount(int count, String partitionId, String errorType, OperationName operationName) { + TestCounter processCounter = meter.getCounters().get("messaging.client.consumed.messages"); + assertNotNull(processCounter); + assertEquals(count > 0 ? 1 : 0, processCounter.getMeasurements().size()); + if (count > 0) { + assertEquals(Long.valueOf(count), processCounter.getMeasurements().get(0).getValue()); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, errorType, operationName, + processCounter.getMeasurements().get(0).getAttributes()); + } + } + + private double getDoubleSeconds(Duration duration) { + return duration.toNanos() / 1_000_000_000.0; + } + + private static class TestCheckpointStore implements CheckpointStore { + private final CheckpointStore inner; + private final Function, Mono> onUpdateCheckpoint; + + TestCheckpointStore(CheckpointStore inner, Function, Mono> onUpdateCheckpoint) { + this.onUpdateCheckpoint = onUpdateCheckpoint; + this.inner = inner; + } + + TestCheckpointStore(Function, Mono> onCheckpoint) { + this(new SampleCheckpointStore(), onCheckpoint); + } + + @Override + public Flux listOwnership(String fullyQualifiedNamespace, String eventHubName, String consumerGroup) { + return inner.listOwnership(fullyQualifiedNamespace, eventHubName, consumerGroup); + } + + @Override + public Flux claimOwnership(List requestedPartitionOwnerships) { + return inner.claimOwnership(requestedPartitionOwnerships); + } + + @Override + public Flux listCheckpoints(String fullyQualifiedNamespace, String eventHubName, String consumerGroup) { + return inner.listCheckpoints(fullyQualifiedNamespace, eventHubName, consumerGroup); + } + + @Override + public Mono updateCheckpoint(Checkpoint checkpoint) { + return onUpdateCheckpoint.apply(inner.updateCheckpoint(checkpoint)); + } + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracerTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracerTests.java index c055307dd963b..d173aa20852fb 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracerTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracerTests.java @@ -11,6 +11,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.Exceptions; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; @@ -30,8 +31,8 @@ public void testSpanEndNoError() { Tracer inner = mock(Tracer.class); when(inner.isEnabled()).thenReturn(true); - EventHubsTracer tracer = new EventHubsTracer(inner, "fqdn", "entityPath"); - tracer.endSpan(null, Context.NONE, null); + EventHubsTracer tracer = new EventHubsTracer(inner, "fqdn", "entityPath", null); + tracer.endSpan(null, null, Context.NONE, null); verify(inner, times(1)).end(isNull(), isNull(), same(Context.NONE)); } @@ -41,9 +42,9 @@ public void testSpanEndNoErrorAndScope() { Tracer inner = mock(Tracer.class); when(inner.isEnabled()).thenReturn(true); - EventHubsTracer tracer = new EventHubsTracer(inner, "fqdn", "entityPath"); + EventHubsTracer tracer = new EventHubsTracer(inner, "fqdn", "entityPath", null); AtomicBoolean closed = new AtomicBoolean(); - tracer.endSpan(null, Context.NONE, () -> closed.set(true)); + tracer.endSpan(null, null, Context.NONE, () -> closed.set(true)); verify(inner, times(1)).end(isNull(), isNull(), same(Context.NONE)); assertTrue(closed.get()); @@ -51,23 +52,29 @@ public void testSpanEndNoErrorAndScope() { @ParameterizedTest @MethodSource("getAmqpException") - public void testSpanEndException(Exception amqpException, String expectedStatus) { + public void testSpanEndException(Exception amqpException, Exception cause, String expectedStatus) { Tracer inner = mock(Tracer.class); when(inner.isEnabled()).thenReturn(true); - EventHubsTracer tracer = new EventHubsTracer(inner, "fqdn", "entityPath"); + EventHubsTracer tracer = new EventHubsTracer(inner, "fqdn", "entityPath", null); - tracer.endSpan(amqpException, Context.NONE, null); + tracer.endSpan(null, amqpException, Context.NONE, null); - verify(inner, times(1)).end(eq(expectedStatus), eq(amqpException), same(Context.NONE)); + verify(inner, times(1)).end(eq(expectedStatus), eq(cause), same(Context.NONE)); } public static Stream getAmqpException() { + RuntimeException runtimeException = new RuntimeException("foo"); + AmqpException amqpNoCauseNoCondition = new AmqpException(false, "foo", null, null); + AmqpException amqpNoCauseCondition = new AmqpException(false, AmqpErrorCondition.NOT_FOUND, "foo", null); + AmqpException amqpNoCauseConditionMessage = new AmqpException(false, AmqpErrorCondition.TIMEOUT_ERROR, "test", null); + AmqpException amqpCauseCondition = new AmqpException(false, AmqpErrorCondition.SERVER_BUSY_ERROR, null, runtimeException, null); return Stream.of( - Arguments.of(new RuntimeException("foo"), null), - Arguments.of(new AmqpException(false, "foo", null, null), null), - Arguments.of(new AmqpException(false, AmqpErrorCondition.NOT_FOUND, "foo", null), AmqpErrorCondition.NOT_FOUND.getErrorCondition()), - Arguments.of(new AmqpException(false, AmqpErrorCondition.TIMEOUT_ERROR, "", null), AmqpErrorCondition.TIMEOUT_ERROR.getErrorCondition()), - Arguments.of(new AmqpException(false, AmqpErrorCondition.SERVER_BUSY_ERROR, null, new RuntimeException("foo"), null), AmqpErrorCondition.SERVER_BUSY_ERROR.getErrorCondition())); + Arguments.of(runtimeException, runtimeException, RuntimeException.class.getName()), + Arguments.of(Exceptions.propagate(runtimeException), runtimeException, RuntimeException.class.getName()), + Arguments.of(amqpNoCauseNoCondition, amqpNoCauseNoCondition, AmqpException.class.getName()), + Arguments.of(amqpNoCauseCondition, amqpNoCauseCondition, amqpNoCauseCondition.getErrorCondition().getErrorCondition()), + Arguments.of(amqpNoCauseConditionMessage, amqpNoCauseConditionMessage, amqpNoCauseConditionMessage.getErrorCondition().getErrorCondition()), + Arguments.of(amqpCauseCondition, runtimeException, amqpCauseCondition.getErrorCondition().getErrorCondition())); } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScopeTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScopeTests.java new file mode 100644 index 0000000000000..b22718332d7e1 --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScopeTests.java @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs.implementation.instrumentation; + +import com.azure.core.test.utils.metrics.TestCounter; +import com.azure.core.test.utils.metrics.TestMeter; +import com.azure.core.util.Context; +import com.azure.core.util.TelemetryAttributes; +import com.azure.core.util.metrics.LongCounter; +import com.azure.core.util.tracing.Tracer; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class InstrumentationScopeTests { + private static final BiConsumer NOOP_METRICS_CALLBACK = (m, s) -> { + }; + + + @Test + public void disabledScope() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(null, null, "fullyQualifiedName", "entityName", "consumerGroup", true); + InstrumentationScope scope = instrumentation.createScope(null); + assertNotNull(scope); + assertFalse(scope.isEnabled()); + assertSame(Context.NONE, scope.getSpan()); + + scope.setCancelled(); + assertNull(scope.getErrorType()); + + scope.setError(new RuntimeException("test")); + assertNull(scope.getError()); + + assertNull(scope.getStartTime()); + + scope.close(); + } + + @Test + public void enabledTracesScope() { + Tracer tracer = mock(Tracer.class); + when(tracer.isEnabled()).thenReturn(true); + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, null, "fullyQualifiedName", "entityName", "consumerGroup", true); + InstrumentationScope scope = instrumentation.createScope(null); + assertNotNull(scope); + assertTrue(scope.isEnabled()); + + assertSame(Context.NONE, scope.getSpan()); + scope.close(); + } + + @Test + public void enabledMetricsScopeNoCallback() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(null, new TestMeter(), "fullyQualifiedName", "entityName", "consumerGroup", true); + InstrumentationScope scope = instrumentation.createScope(null); + assertNotNull(scope); + assertNotNull(scope.getStartTime()); + assertTrue(scope.isEnabled()); + scope.close(); + } + + @Test + public void enabledMetricsScopeWithCallback() { + TestMeter meter = new TestMeter(); + LongCounter counter = meter.createLongCounter("test.me.counter", null, null); + TelemetryAttributes attributes = meter.createAttributes(Collections.emptyMap()); + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(null, meter, "fullyQualifiedName", "entityName", "consumerGroup", true); + + InstrumentationScope scope = instrumentation.createScope((m, s) -> counter.add(1, attributes, s.getSpan())); + scope.setSpan(new Context("foo", "bar")); + assertNotNull(scope); + assertTrue(scope.isEnabled()); + scope.close(); + + TestCounter testCounter = (TestCounter) counter; + assertEquals(1, testCounter.getMeasurements().size()); + assertEquals(1L, testCounter.getMeasurements().get(0).getValue()); + assertSame(scope.getSpan(), testCounter.getMeasurements().get(0).getContext()); + } + + @Test + public void recordsStartTimeWhenMetricsEnabledScope() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(null, new TestMeter(), "fullyQualifiedName", "entityName", "consumerGroup", true); + InstrumentationScope scope = instrumentation.createScope(null); + assertNotNull(scope.getStartTime()); + scope.close(); + } + + @Test + public void scopeClosesSpan() { + Tracer tracer = mock(Tracer.class); + when(tracer.isEnabled()).thenReturn(true); + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, null, "fullyQualifiedName", "entityName", "consumerGroup", true); + + InstrumentationScope scope = instrumentation.createScope(null); + scope.close(); + + verify(tracer).end(isNull(), isNull(), same(Context.NONE)); + } + + @Test + public void scopeClosesSpanAndScope() { + Tracer tracer = mock(Tracer.class); + when(tracer.isEnabled()).thenReturn(true); + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, null, "fullyQualifiedName", "entityName", "consumerGroup", true); + + AtomicBoolean spanClosed = new AtomicBoolean(); + when(tracer.makeSpanCurrent(any(Context.class))).thenReturn(() -> spanClosed.set(true)); + + AtomicBoolean metricCallbackCalled = new AtomicBoolean(); + InstrumentationScope scope = instrumentation.createScope((m, s) -> metricCallbackCalled.set(true)) + .makeSpanCurrent(); + + scope.close(); + + assertTrue(spanClosed.get()); + assertTrue(metricCallbackCalled.get()); + verify(tracer).end(isNull(), isNull(), same(Context.NONE)); + } + + @Test + public void setError() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(null, new TestMeter(), "fullyQualifiedName", "entityName", "consumerGroup", true); + + InstrumentationScope scope = instrumentation.createScope(null); + Throwable error = new RuntimeException("test"); + scope.setError(error); + assertSame(error, scope.getError()); + assertEquals(RuntimeException.class.getName(), scope.getErrorType()); + } + + @Test + public void setCancelled() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(null, new TestMeter(), "fullyQualifiedName", "entityName", "consumerGroup", true); + + InstrumentationScope scope = instrumentation.createScope(null); + scope.setCancelled(); + assertNull(scope.getError()); + assertEquals("cancelled", scope.getErrorType()); + + Throwable error = new RuntimeException("test"); + scope.setError(error); + + assertEquals("cancelled", scope.getErrorType()); + assertSame(error, scope.getError()); + } +} From a7983e9808b0e32b841bc2c8921ac7360afdf306 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 10 Sep 2024 18:31:06 -0700 Subject: [PATCH 02/21] generic operation duration and nits --- .../EventHubConsumerAsyncClient.java | 9 +- .../eventhubs/EventHubConsumerClient.java | 4 +- .../EventHubProducerAsyncClient.java | 4 +- .../EventHubsProducerInstrumentation.java | 29 ++++-- .../eventhubs/MessageFluxWrapper.java | 5 +- .../eventhubs/SynchronousReceiver.java | 3 +- .../EventHubsConsumerInstrumentation.java | 36 ++++++-- .../EventHubsMetricsProvider.java | 15 ++++ .../instrumentation/EventHubsTracer.java | 45 ++++------ .../instrumentation/InstrumentationScope.java | 3 +- .../instrumentation/InstrumentationUtils.java | 5 +- .../InstrumentedCheckpointStore.java | 4 +- .../InstrumentedMessageFlux.java | 89 +++++++++++++++++++ .../EventHubProducerAsyncClientTest.java | 61 +++++++++++-- .../eventhubs/TracingIntegrationTests.java | 63 +++++++++---- ...EventHubsConsumerInstrumentationTests.java | 75 +++++++++++----- 16 files changed, 346 insertions(+), 104 deletions(-) create mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedMessageFlux.java diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClient.java index f45b50cc06b0c..a8b75330314e8 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerAsyncClient.java @@ -18,6 +18,7 @@ import com.azure.messaging.eventhubs.implementation.AmqpReceiveLinkProcessor; import com.azure.messaging.eventhubs.implementation.EventHubManagementNode; import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; +import com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentedMessageFlux; import com.azure.messaging.eventhubs.models.EventPosition; import com.azure.messaging.eventhubs.models.PartitionEvent; import com.azure.messaging.eventhubs.models.ReceiveOptions; @@ -308,7 +309,7 @@ boolean isV2() { */ @ServiceMethod(returns = ReturnType.SINGLE) public Mono getEventHubProperties() { - return instrumentation.getTracer().traceMono(connectionProcessor.getManagementNodeWithRetries() + return instrumentation.instrumentMono(connectionProcessor.getManagementNodeWithRetries() .flatMap(EventHubManagementNode::getEventHubProperties), GET_EVENT_HUB_PROPERTIES, null); } @@ -341,7 +342,7 @@ public Mono getPartitionProperties(String partitionId) { return monoError(LOGGER, new IllegalArgumentException("'partitionId' cannot be an empty string.")); } - return instrumentation.getTracer().traceMono( + return instrumentation.instrumentMono( connectionProcessor.getManagementNodeWithRetries().flatMap(node -> node.getPartitionProperties(partitionId)), GET_PARTITION_PROPERTIES, partitionId); } @@ -589,8 +590,8 @@ private EventHubPartitionAsyncConsumer createPartitionConsumer(String linkName, final MessageFluxWrapper linkMessageProcessor; if (connectionProcessor.isV2()) { - final MessageFlux messageFlux = new MessageFlux(receiveLinkFlux, prefetchCount, CreditFlowMode.EmissionDriven, MessageFlux.NULL_RETRY_POLICY); - linkMessageProcessor = new MessageFluxWrapper(messageFlux); + MessageFlux messageFlux = new MessageFlux(receiveLinkFlux, prefetchCount, CreditFlowMode.EmissionDriven, MessageFlux.NULL_RETRY_POLICY); + linkMessageProcessor = new MessageFluxWrapper(InstrumentedMessageFlux.instrument(messageFlux, partitionId, instrumentation)); } else { final AmqpReceiveLinkProcessor receiveLinkProcessor = receiveLinkFlux.subscribeWith( new AmqpReceiveLinkProcessor(entityPath, prefetchCount, partitionId, connectionProcessor, instrumentation)); diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java index e36e14320674a..c77b944761e5d 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java @@ -260,7 +260,7 @@ public IterableStream receiveFromPartition(String partitionId, i } if (consumer.isV2()) { - // TODO(limolkova) instrument + // Sync receiver instrumentation is implemented in the SynchronousReceiver class return syncReceiver.receive(partitionId, startingPosition, defaultReceiveOptions, maximumMessageCount, maximumWaitTime); } @@ -315,7 +315,7 @@ public IterableStream receiveFromPartition(String partitionId, i } if (consumer.isV2()) { - // TODO (instrument) + // Sync receiver instrumentation is implemented in the SynchronousReceiver class return syncReceiver.receive(partitionId, startingPosition, receiveOptions, maximumMessageCount, maximumWaitTime); } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClient.java index 79c65e1db2617..a842aaa2909ad 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClient.java @@ -299,7 +299,7 @@ public String getEventHubName() { */ @ServiceMethod(returns = ReturnType.SINGLE) public Mono getEventHubProperties() { - return instrumentation.getTracer().traceMono( + return instrumentation.instrumentMono( connectionProcessor.getManagementNodeWithRetries().flatMap(EventHubManagementNode::getEventHubProperties), GET_EVENT_HUB_PROPERTIES, null); } @@ -324,7 +324,7 @@ public Flux getPartitionIds() { */ @ServiceMethod(returns = ReturnType.SINGLE) public Mono getPartitionProperties(String partitionId) { - return instrumentation.getTracer().traceMono( + return instrumentation.instrumentMono( connectionProcessor.getManagementNodeWithRetries().flatMap(node -> node.getPartitionProperties(partitionId)), GET_PARTITION_PROPERTIES, partitionId); } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java index 18588abff5978..6fa76e15dc8b3 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java @@ -5,7 +5,6 @@ import com.azure.core.util.Context; import com.azure.core.util.metrics.Meter; -import com.azure.core.util.tracing.SpanKind; import com.azure.core.util.tracing.StartSpanOptions; import com.azure.core.util.tracing.Tracer; import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsMetricsProvider; @@ -17,8 +16,11 @@ import java.util.function.BiConsumer; +import static com.azure.core.util.tracing.SpanKind.CLIENT; +import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_BATCH_MESSAGE_COUNT; import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_PARTITION_ID; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.SEND; class EventHubsProducerInstrumentation { private final EventHubsTracer tracer; @@ -39,7 +41,7 @@ Mono sendBatch(Mono publisher, EventDataBatch batch) { return Mono.using( () -> new InstrumentationScope(tracer, meter, reportMetricsCallback) - .setSpan(startPublishSpanWithLinks(batch, Context.NONE)), + .setSpan(startPublishSpanWithLinks(batch)), scope -> publisher .doOnError(scope::setError) .doOnCancel(scope::setCancelled), @@ -50,12 +52,27 @@ public EventHubsTracer getTracer() { return tracer; } - private Context startPublishSpanWithLinks(EventDataBatch batch, Context context) { + public Mono instrumentMono(Mono publisher, OperationName operationName, String partitionId) { + if (!isEnabled()) { + return publisher; + } + + return Mono.using( + () -> new InstrumentationScope(tracer, meter, (m, s) -> m.reportGenericOperationDuration(operationName, partitionId, s)) + .setSpan(tracer.startGenericOperationSpan(operationName, partitionId, Context.NONE)), + scope -> publisher + .doOnError(scope::setError) + .doOnCancel(scope::setCancelled) + .contextWrite(c -> c.put(PARENT_TRACE_CONTEXT_KEY, scope.getSpan())), + InstrumentationScope::close); + } + + private Context startPublishSpanWithLinks(EventDataBatch batch) { if (!tracer.isEnabled()) { - return context; + return Context.NONE; } - StartSpanOptions startOptions = tracer.createStartOptions(SpanKind.CLIENT, OperationName.SEND, null); + StartSpanOptions startOptions = tracer.createStartOptions(CLIENT, SEND, null); if (batch != null) { startOptions.setAttribute(MESSAGING_BATCH_MESSAGE_COUNT, batch.getCount()); if (batch.getPartitionId() != null) { @@ -66,7 +83,7 @@ private Context startPublishSpanWithLinks(EventDataBatch batch, Context context) } } - return tracer.startSpan(OperationName.SEND, startOptions, context); + return tracer.startSpan(SEND, startOptions, Context.NONE); } private boolean isEnabled() { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java index edab872b8ed75..cc26669c98794 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java @@ -3,7 +3,6 @@ package com.azure.messaging.eventhubs; -import com.azure.core.amqp.implementation.MessageFlux; import com.azure.messaging.eventhubs.implementation.AmqpReceiveLinkProcessor; import org.apache.qpid.proton.message.Message; import reactor.core.publisher.Flux; @@ -12,10 +11,10 @@ final class MessageFluxWrapper { private final AmqpReceiveLinkProcessor receiveLinkProcessor; - private final MessageFlux messageFlux; + private final Flux messageFlux; private final boolean isV2; - MessageFluxWrapper(MessageFlux messageFlux) { + MessageFluxWrapper(Flux messageFlux) { this.messageFlux = Objects.requireNonNull(messageFlux, "'messageFlux' cannot be null."); this.receiveLinkProcessor = null; this.isV2 = true; diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java index 745663c20fde0..6f013e76ac5a6 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java @@ -132,8 +132,7 @@ private WindowedSubscriber createSubscriber(String partitionId) final WindowedSubscriberOptions options = new WindowedSubscriberOptions<>(); options.setWindowDecorator(toDecorate -> { // Decorates the provided 'toDecorate' flux for tracing the signals (events, termination) it produces. - return toDecorate; - //return instrumentation.reportSyncReceiveSpan(SYNC_RECEIVE_SPAN_NAME, startTime, toDecorate, Context.NONE); + return instrumentation.syncReceive(toDecorate, partitionId); }); return new WindowedSubscriber<>(Collections.singletonMap(PARTITION_ID_KEY, partitionId), TERMINAL_MESSAGE, options); } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java index af2ea5e673ebd..08fdda39a71eb 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java @@ -16,12 +16,14 @@ import org.apache.qpid.proton.amqp.messaging.ApplicationProperties; import org.apache.qpid.proton.message.Message; import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; import java.time.Instant; import java.util.function.BiConsumer; import static com.azure.core.amqp.AmqpMessageConstant.ENQUEUED_TIME_UTC_ANNOTATION_NAME; -import static com.azure.core.util.tracing.SpanKind.CONSUMER; +import static com.azure.core.util.tracing.SpanKind.CLIENT; +import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_BATCH_MESSAGE_COUNT; import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_PARTITION_ID; import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.RECEIVE; @@ -39,6 +41,7 @@ public EventHubsConsumerInstrumentation(Tracer tracer, Meter meter, String fully this.isSync = isSyncConsumer; } + public EventHubsTracer getTracer() { return tracer; } @@ -52,7 +55,7 @@ public InstrumentationScope startAsyncConsume(Message message, String partitionI return NOOP_SCOPE; } - InstrumentationScope scope = createScope((m, s) -> { + InstrumentationScope scope = createScope((m, s) -> { if (!isSync) { m.reportProcess(1, partitionId, s); } @@ -62,8 +65,8 @@ public InstrumentationScope startAsyncConsume(Message message, String partitionI ApplicationProperties properties = message.getApplicationProperties(); scope.setSpan(tracer.startProcessSpan(properties == null ? null : properties.getValue(), enqueuedTime, - partitionId, - Context.NONE)) + partitionId + )) .makeSpanCurrent(); } @@ -78,7 +81,7 @@ public Flux syncReceive(Flux events, String part return events; } - StartSpanOptions startOptions = tracer.isEnabled() ? tracer.createStartOptions(CONSUMER, RECEIVE, partitionId) : null; + StartSpanOptions startOptions = tracer.isEnabled() ? tracer.createStartOptions(CLIENT, RECEIVE, partitionId) : null; Integer[] receivedCount = new Integer[]{0}; return Flux.using( @@ -119,7 +122,7 @@ public InstrumentationScope startProcess(EventBatchContext batchContext) { m.reportProcess(batchContext.getEvents().size(), batchContext.getPartitionContext().getPartitionId(), s)); return scope - .setSpan(tracer.startProcessSpan(batchContext, Context.NONE)) + .setSpan(tracer.startProcessSpan(batchContext)) .makeSpanCurrent(); } @@ -134,15 +137,30 @@ public InstrumentationScope startProcess(EventContext eventContext) { Context span = tracer.startProcessSpan(event.getProperties(), event.getEnqueuedTime(), - eventContext.getPartitionContext().getPartitionId(), - Context.NONE); + eventContext.getPartitionContext().getPartitionId() + ); return scope .setSpan(span) .makeSpanCurrent(); } - boolean isEnabled() { + public Mono instrumentMono(Mono publisher, OperationName operationName, String partitionId) { + if (!isEnabled()) { + return publisher; + } + + return Mono.using( + () -> createScope((m, s) -> m.reportGenericOperationDuration(operationName, partitionId, s)) + .setSpan(tracer.startGenericOperationSpan(operationName, partitionId, Context.NONE)), + scope -> publisher + .doOnError(scope::setError) + .doOnCancel(scope::setCancelled) + .contextWrite(c -> c.put(PARENT_TRACE_CONTEXT_KEY, scope.getSpan())), + InstrumentationScope::close); + } + + public boolean isEnabled() { return tracer.isEnabled() || meter.isEnabled(); } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java index c61cde3aad20d..fa1a46d0c987d 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java @@ -46,6 +46,8 @@ public class EventHubsMetricsProvider { private AttributeCache receiveAttributeCacheSuccess; private AttributeCache checkpointAttributeCacheSuccess; private AttributeCache processAttributeCacheSuccess; + private AttributeCache getPartitionPropertiesAttributeCacheSuccess; + private AttributeCache getEventHubPropertiesAttributeCacheSuccess; private AttributeCache lagAttributeCache; private LongCounter publishedEventCounter; private LongCounter consumedEventCounter; @@ -61,6 +63,8 @@ public EventHubsMetricsProvider(Meter meter, String namespace, String entityName this.receiveAttributeCacheSuccess = AttributeCache.create(meter, RECEIVE, commonAttributes); this.checkpointAttributeCacheSuccess = AttributeCache.create(meter, CHECKPOINT, commonAttributes); this.processAttributeCacheSuccess = AttributeCache.create(meter, PROCESS, commonAttributes); + this.getPartitionPropertiesAttributeCacheSuccess = AttributeCache.create(meter, OperationName.GET_PARTITION_PROPERTIES, commonAttributes); + this.getEventHubPropertiesAttributeCacheSuccess = AttributeCache.create(meter, OperationName.GET_EVENT_HUB_PROPERTIES, commonAttributes); this.lagAttributeCache = new AttributeCache(meter, MESSAGING_DESTINATION_PARTITION_ID, commonAttributes); this.publishedEventCounter = meter.createLongCounter(MESSAGING_CLIENT_PUBLISHED_MESSAGES, "The number of published events", "{event}"); @@ -120,6 +124,13 @@ public void reportCheckpoint(Checkpoint checkpoint, InstrumentationScope scope) } } + public void reportGenericOperationDuration(OperationName operationName, String partitionId, InstrumentationScope scope) { + if (isEnabled && operationDuration.isEnabled()) { + operationDuration.record(getDurationInSeconds(scope.getStartTime()), + getOrCreateAttributes(operationName, partitionId, scope.getErrorType()), scope.getSpan()); + } + } + private TelemetryAttributes getOrCreateAttributes(OperationName operationName, String partitionId, String errorType) { if (errorType == null) { switch (operationName) { @@ -131,6 +142,10 @@ private TelemetryAttributes getOrCreateAttributes(OperationName operationName, S return checkpointAttributeCacheSuccess.getOrCreate(partitionId); case PROCESS: return processAttributeCacheSuccess.getOrCreate(partitionId); + case GET_PARTITION_PROPERTIES: + return getPartitionPropertiesAttributeCacheSuccess.getOrCreate(partitionId); + case GET_EVENT_HUB_PROPERTIES: + return getEventHubPropertiesAttributeCacheSuccess.getOrCreate(partitionId); default: LOGGER.atVerbose() .addKeyValue("operationName", operationName) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java index a102d3e0a5683..b1de58cba414f 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java @@ -13,7 +13,6 @@ import com.azure.core.util.tracing.TracingLink; import com.azure.messaging.eventhubs.EventData; import com.azure.messaging.eventhubs.models.EventBatchContext; -import reactor.core.publisher.Mono; import java.time.Instant; import java.time.ZoneOffset; @@ -22,7 +21,10 @@ import java.util.Objects; import java.util.Optional; -import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; +import static com.azure.core.util.tracing.SpanKind.CLIENT; +import static com.azure.core.util.tracing.SpanKind.CONSUMER; +import static com.azure.core.util.tracing.SpanKind.PRODUCER; + import static com.azure.core.util.tracing.Tracer.SPAN_CONTEXT_KEY; import static com.azure.messaging.eventhubs.EventHubClientBuilder.DEFAULT_CONSUMER_GROUP_NAME; import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.DIAGNOSTIC_ID_KEY; @@ -73,21 +75,14 @@ public Context startSpan(OperationName operationName, StartSpanOptions startOpti return isEnabled() ? tracer.start(getSpanName(operationName), startOptions, context) : context; } - public Mono traceMono(Mono publisher, OperationName operationName, String partitionId) { + public Context startGenericOperationSpan(OperationName operationName, String partitionId, Context parent) { if (!isEnabled()) { - return publisher; + return parent; } - return Mono.using( - () -> new InstrumentationScope(this, null, null) - .setSpan(tracer.start(getSpanName(operationName), - createStartOptions(SpanKind.CLIENT, operationName, partitionId), - Context.NONE)), - scope -> publisher - .doOnError(scope::setError) - .doOnCancel(scope::setCancelled) - .contextWrite(c -> c.put(PARENT_TRACE_CONTEXT_KEY, scope.getSpan())), - InstrumentationScope::close); + return tracer.start(getSpanName(operationName), + createStartOptions(CLIENT, operationName, partitionId), + parent); } /** @@ -106,7 +101,7 @@ public void reportMessageSpan(EventData eventData, Context eventContext) { } // Starting the span makes the sampling decision (nothing is logged at this time) - StartSpanOptions startOptions = createStartOptions(SpanKind.PRODUCER, EVENT, null); + StartSpanOptions startOptions = createStartOptions(PRODUCER, EVENT, null); Context eventSpanContext = tracer.start(getSpanName(EVENT), startOptions, eventContext); @@ -133,9 +128,7 @@ public void reportMessageSpan(EventData eventData, Context eventContext) { tracer.end(error, exception, eventSpanContext); Optional spanContext = eventSpanContext.getData(SPAN_CONTEXT_KEY); - if (spanContext.isPresent()) { - eventData.addContext(SPAN_CONTEXT_KEY, spanContext.get()); - } + spanContext.ifPresent(o -> eventData.addContext(SPAN_CONTEXT_KEY, o)); } public TracingLink createLink(Map applicationProperties, Instant enqueuedTime) { @@ -189,10 +182,10 @@ AutoCloseable makeSpanCurrent(Context context) { return isEnabled() ? tracer.makeSpanCurrent(context) : NOOP_AUTOCLOSEABLE; } - Context startProcessSpan(Map applicationProperties, Instant enqueuedTime, String partitionId, Context parent) { + Context startProcessSpan(Map applicationProperties, Instant enqueuedTime, String partitionId) { if (isEnabled()) { Context remoteContext = extractContext(applicationProperties); - StartSpanOptions startOptions = createStartOptions(SpanKind.CONSUMER, PROCESS, partitionId) + StartSpanOptions startOptions = createStartOptions(CONSUMER, PROCESS, partitionId) .addLink(createLink(remoteContext, null)) .setRemoteParent(remoteContext); @@ -201,15 +194,15 @@ Context startProcessSpan(Map applicationProperties, Instant enqu startOptions.setAttribute(MESSAGING_EVENTHUBS_MESSAGE_ENQUEUED_TIME, enqueuedTime.atOffset(ZoneOffset.UTC).toEpochSecond()); } - return tracer.start(getSpanName(PROCESS), startOptions, parent); + return tracer.start(getSpanName(PROCESS), startOptions, Context.NONE); } - return parent; + return Context.NONE; } - Context startProcessSpan(EventBatchContext batchContext, Context parent) { + Context startProcessSpan(EventBatchContext batchContext) { if (isEnabled() && batchContext != null && !CoreUtils.isNullOrEmpty(batchContext.getEvents())) { - StartSpanOptions startOptions = createStartOptions(SpanKind.CONSUMER, PROCESS, + StartSpanOptions startOptions = createStartOptions(CONSUMER, PROCESS, batchContext.getPartitionContext().getPartitionId()); startOptions.setAttribute(MESSAGING_BATCH_MESSAGE_COUNT, batchContext.getEvents().size()); @@ -217,10 +210,10 @@ Context startProcessSpan(EventBatchContext batchContext, Context parent) { startOptions.addLink(createLink(event.getProperties(), event.getEnqueuedTime())); } - return tracer.start(getSpanName(PROCESS), startOptions, parent); + return tracer.start(getSpanName(PROCESS), startOptions, Context.NONE); } - return parent; + return Context.NONE; } public StartSpanOptions createStartOptions(SpanKind kind, OperationName operationName, String partitionId) { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java index b8132540d210e..3b3987f32193f 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java @@ -17,12 +17,12 @@ public final class InstrumentationScope implements AutoCloseable { private final EventHubsMetricsProvider meter; private final EventHubsTracer tracer; private final boolean isEnabled; + private final BiConsumer reportMetricsCallback; private Instant startTime; private Throwable error; private String errorType; private Context span = Context.NONE; private AutoCloseable spanScope; - private BiConsumer reportMetricsCallback; private boolean closed = false; public InstrumentationScope(EventHubsTracer tracer, @@ -66,6 +66,7 @@ public InstrumentationScope setCancelled() { if (isEnabled && error == null) { errorType = CANCELLED_ERROR_TYPE_VALUE; } + return this; } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java index e4ea48bd84702..d3f6521f9fa0a 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java @@ -37,9 +37,6 @@ public final class InstrumentationUtils { public static final String MESSAGING_SYSTEM_VALUE = "eventhubs"; public static final String CANCELLED_ERROR_TYPE_VALUE = "cancelled"; - // _OTHER is a magic string defined in OpenTelemetry for 'unknown' errors - public static final String OTHER_ERROR_TYPE_VALUE = "_OTHER"; - // context propagation constants public static final String TRACEPARENT_KEY = "traceparent"; public static final String DIAGNOSTIC_ID_KEY = "Diagnostic-Id"; @@ -64,7 +61,7 @@ public static String getErrorType(Throwable error) { return ((AmqpException) error).getErrorCondition().getErrorCondition(); } - return (error != null) ? error.getClass().getName() : OTHER_ERROR_TYPE_VALUE; + return error.getClass().getName(); } public static Throwable unwrap(Throwable error) { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedCheckpointStore.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedCheckpointStore.java index 0685847195287..a723feac57c99 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedCheckpointStore.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedCheckpointStore.java @@ -4,7 +4,6 @@ package com.azure.messaging.eventhubs.implementation.instrumentation; import com.azure.core.util.Context; -import com.azure.core.util.tracing.SpanKind; import com.azure.messaging.eventhubs.CheckpointStore; import com.azure.messaging.eventhubs.models.Checkpoint; import com.azure.messaging.eventhubs.models.PartitionOwnership; @@ -13,6 +12,7 @@ import java.util.List; +import static com.azure.core.util.tracing.SpanKind.INTERNAL; import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.CHECKPOINT; public final class InstrumentedCheckpointStore implements CheckpointStore { @@ -62,7 +62,7 @@ public Mono updateCheckpoint(Checkpoint checkpoint) { private Context startSpan(String partitionId) { return tracer.isEnabled() - ? tracer.startSpan(CHECKPOINT, tracer.createStartOptions(SpanKind.INTERNAL, CHECKPOINT, partitionId), Context.NONE) + ? tracer.startSpan(CHECKPOINT, tracer.createStartOptions(INTERNAL, CHECKPOINT, partitionId), Context.NONE) : Context.NONE; } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedMessageFlux.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedMessageFlux.java new file mode 100644 index 0000000000000..8a4fe69f7597d --- /dev/null +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentedMessageFlux.java @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.messaging.eventhubs.implementation.instrumentation; + +import com.azure.core.amqp.implementation.MessageFlux; +import org.apache.qpid.proton.message.Message; +import org.reactivestreams.Subscription; +import reactor.core.CoreSubscriber; +import reactor.core.publisher.BaseSubscriber; +import reactor.core.publisher.Flux; +import reactor.core.publisher.FluxOperator; + +import java.util.Objects; + +/** + * Flux operator that instruments receive callbacks on async client + */ +public final class InstrumentedMessageFlux extends FluxOperator { + private final EventHubsConsumerInstrumentation instrumentation; + private final String partitionId; + + private InstrumentedMessageFlux(MessageFlux upstream, String partitionId, EventHubsConsumerInstrumentation instrumentation) { + super(upstream); + this.instrumentation = instrumentation; + this.partitionId = partitionId; + } + + public static Flux instrument(MessageFlux source, String partitionId, EventHubsConsumerInstrumentation instrumentation) { + if (instrumentation.isEnabled()) { + return new InstrumentedMessageFlux(source, partitionId, instrumentation); + } + + return source; + } + + @Override + public void subscribe(CoreSubscriber coreSubscriber) { + Objects.requireNonNull(coreSubscriber, "'coreSubscriber' cannot be null."); + + source.subscribe(new TracingSubscriber(coreSubscriber, partitionId, instrumentation)); + } + + private static class TracingSubscriber extends BaseSubscriber { + + private final CoreSubscriber downstream; + private final EventHubsConsumerInstrumentation instrumentation; + private final String partitionId; + + TracingSubscriber(CoreSubscriber downstream, String partitionId, EventHubsConsumerInstrumentation instrumentation) { + this.downstream = downstream; + this.instrumentation = instrumentation; + this.partitionId = partitionId; + } + + @Override + public reactor.util.context.Context currentContext() { + return downstream.currentContext(); + } + + @Override + protected void hookOnSubscribe(Subscription subscription) { + downstream.onSubscribe(this); + } + + @Override + protected void hookOnNext(Message message) { + InstrumentationScope scope = instrumentation.startAsyncConsume(message, partitionId); + try { + downstream.onNext(message); + } catch (Exception e) { + scope.setError(e); + throw e; + } finally { + scope.close(); + } + } + + @Override + protected void hookOnError(Throwable throwable) { + downstream.onError(throwable); + } + + @Override + protected void hookOnComplete() { + downstream.onComplete(); + } + } +} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClientTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClientTest.java index 5a618f4edb226..3bba376003922 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClientTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventHubProducerAsyncClientTest.java @@ -35,6 +35,7 @@ import com.azure.messaging.eventhubs.implementation.EventHubConnectionProcessor; import com.azure.messaging.eventhubs.implementation.EventHubManagementNode; import com.azure.messaging.eventhubs.implementation.EventHubReactorAmqpConnection; +import com.azure.messaging.eventhubs.implementation.instrumentation.OperationName; import com.azure.messaging.eventhubs.models.CreateBatchOptions; import com.azure.messaging.eventhubs.models.SendOptions; import org.apache.qpid.proton.Proton; @@ -84,6 +85,7 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.azure.core.amqp.AmqpMessageConstant.ENQUEUED_TIME_UTC_ANNOTATION_NAME; @@ -1099,7 +1101,7 @@ void sendsAnEventDataBatchWithMetricsFailure(Throwable error, String expectedErr .verify(DEFAULT_TIMEOUT); assertSendCount(meter, null, 1, expectedErrorType, null); - assertSendDuration(meter, null, null, expectedErrorType, null); + assertOperationDuration(meter, SEND, null, null, expectedErrorType, null); } @Test @@ -1124,7 +1126,47 @@ void sendsAnEventDataBatchWithMetricsPartitionId() { .verify(DEFAULT_TIMEOUT); assertSendCount(meter, partitionId, 1, null, null); - assertSendDuration(meter, partitionId, null, null, null); + assertOperationDuration(meter, SEND, partitionId, null, null, null); + } + + @Test + void getPropertiesReportMetrics() { + String partitionId = "1"; + String entityPath = EVENT_HUB_NAME + "/Partitions/" + partitionId; + when(connection.createSendLink(eq(entityPath), eq(entityPath), any(), any())).thenReturn(Mono.just(sendLink)); + when(sendLink.send(anyList())).thenReturn(Mono.empty()); + when(sendLink.getHostname()).thenReturn(HOSTNAME); + when(sendLink.getEntityPath()).thenReturn(entityPath); + when(sendLink.getLinkName()).thenReturn(entityPath); + when(sendLink.send(any(Message.class))).thenReturn(Mono.empty()); + + EventHubManagementNode managementNode = mock(EventHubManagementNode.class); + EventHubProperties ehProperties = new EventHubProperties(EVENT_HUB_NAME, Instant.now(), new String[]{partitionId}); + PartitionProperties partitionProperties = new PartitionProperties(EVENT_HUB_NAME, partitionId, + 1L, 2L, OffsetDateTime.now().toString(), Instant.now(), false); + + when(connection.getManagementNode()).thenReturn(Mono.just(managementNode)); + when(managementNode.getEventHubProperties()).thenReturn(Mono.just(ehProperties)); + when(managementNode.getPartitionProperties(anyString())).thenReturn(Mono.just(partitionProperties)); + + TestMeter meter = new TestMeter(); + EventHubsProducerInstrumentation instrumentation = new EventHubsProducerInstrumentation(null, meter, HOSTNAME, EVENT_HUB_NAME); + EventHubProducerAsyncClient producer = new EventHubProducerAsyncClient(HOSTNAME, EVENT_HUB_NAME, + connectionProcessor, retryOptions, messageSerializer, testScheduler, false, onClientClosed, CLIENT_IDENTIFIER, instrumentation); + + StepVerifier.create(producer.getPartitionProperties(partitionId)) + .expectNextCount(1) + .expectComplete() + .verify(DEFAULT_TIMEOUT); + + assertOperationDuration(meter, GET_PARTITION_PROPERTIES, partitionId, null, null, null); + + StepVerifier.create(producer.getEventHubProperties()) + .expectNextCount(1) + .expectComplete() + .verify(DEFAULT_TIMEOUT); + + assertOperationDuration(meter, GET_EVENT_HUB_PROPERTIES, null, null, null, null); } @Test @@ -1174,7 +1216,7 @@ void sendsAnEventDataBatchWithMetricsAndTraces() { .verify(DEFAULT_TIMEOUT); assertSendCount(meter, null, 1, null, "parent span"); - assertSendDuration(meter, null, null, null, "parent span"); + assertOperationDuration(meter, SEND, null, null, null, "parent span"); } @Test @@ -1625,17 +1667,20 @@ private void assertSendCount(TestMeter meter, String partitionId, int expectedVa } } - private void assertSendDuration(TestMeter meter, String partitionId, Double expectedValue, String expectedErrorType, Object parentContext) { - TestHistogram sendDuration = meter.getHistograms().get("messaging.client.operation.duration"); - assertNotNull(sendDuration); - List> measurements = sendDuration.getMeasurements(); + private void assertOperationDuration(TestMeter meter, OperationName operation, String partitionId, Double expectedValue, String expectedErrorType, Object parentContext) { + TestHistogram duration = meter.getHistograms().get("messaging.client.operation.duration"); + assertNotNull(duration); + List> measurements = duration.getMeasurements() + .stream() + .filter(m -> operation.toString().equals(m.getAttributes().get("messaging.operation.name"))) + .collect(Collectors.toList()); assertEquals(1, measurements.size()); if (expectedValue != null) { assertEquals(expectedValue, measurements.get(0).getValue(), expectedValue); } assertAllAttributes(HOSTNAME, EVENT_HUB_NAME, partitionId, null, expectedErrorType, - SEND, measurements.get(0).getAttributes()); + operation, measurements.get(0).getAttributes()); if (parentContext != null) { assertEquals(parentContext, measurements.get(0).getContext().getData(PARENT_TRACE_CONTEXT_KEY).get()); } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index 68e653d52bf32..021d96ecf438b 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -6,6 +6,8 @@ import com.azure.core.credential.TokenCredential; import com.azure.core.tracing.opentelemetry.OpenTelemetryTracingOptions; import com.azure.core.util.ClientOptions; +import com.azure.core.util.Configuration; +import com.azure.core.util.ConfigurationBuilder; import com.azure.core.util.TracingOptions; import com.azure.core.util.logging.ClientLogger; import com.azure.messaging.eventhubs.implementation.instrumentation.OperationName; @@ -26,11 +28,13 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.data.SpanData; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.api.parallel.Isolated; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; import reactor.test.StepVerifier; @@ -65,7 +69,6 @@ @Isolated("Sets global TracingProvider.") @Execution(ExecutionMode.SAME_THREAD) -@Disabled("Tracing tests need to be disabled until the discrepancy with the core is resolved.") public class TracingIntegrationTests extends IntegrationTestBase { private static final byte[] CONTENTS_BYTES = "Some-contents".getBytes(StandardCharsets.UTF_8); private static final String PARTITION_ID = "0"; @@ -91,38 +94,54 @@ protected void beforeTest() { GlobalOpenTelemetry.resetForTest(); spanProcessor = toClose(new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), testName)); - OpenTelemetrySdk.builder() + OpenTelemetrySdk otel = OpenTelemetrySdk.builder() .setTracerProvider( SdkTracerProvider.builder() .addSpanProcessor(spanProcessor) .build()) .buildAndRegisterGlobal(); - createClients(null); + createClients(otel, null); testStartTime = Instant.now().minusSeconds(1); data = new EventData(CONTENTS_BYTES); } - private void createClients(OpenTelemetrySdk otel) { + private Configuration createConfiguration(String v2Stack) { + ConfigurationBuilder configBuilder = new ConfigurationBuilder(); + + if (v2Stack == null) { + return configBuilder.build(); + } + + return configBuilder + .putProperty("com.azure.messaging.eventhubs.v2", v2Stack) + .build(); + } + + private void createClients(OpenTelemetrySdk otel, String v2Stack) { dispose(); ClientOptions options = new ClientOptions(); if (otel != null) { options.setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); } + Configuration config = createConfiguration(v2Stack); producer = toClose(createBuilder() .clientOptions(options) + .configuration(config) .buildAsyncProducerClient()); consumer = toClose(createBuilder() .clientOptions(options) + .configuration(config) .consumerGroup("$Default") .buildAsyncConsumerClient()); consumerSync = toClose(createBuilder() .clientOptions(options) + .configuration(config) .consumerGroup("$Default") .buildConsumerClient()); } @@ -132,11 +151,15 @@ protected void afterTest() { GlobalOpenTelemetry.resetForTest(); } - @Test - public void sendAndReceiveFromPartition() throws InterruptedException { + @ParameterizedTest + @ValueSource(strings = {"true", "false"}) + @NullSource + public void sendAndReceiveFromPartition(String v2) throws InterruptedException { AtomicReference receivedMessage = new AtomicReference<>(); AtomicReference receivedSpan = new AtomicReference<>(); + createClients(null, v2); + CountDownLatch latch = new CountDownLatch(2); spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer @@ -176,7 +199,7 @@ public void sendAndReceive() throws InterruptedException { spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer .receive() - .take(1) + .take(DEFAULT_TIMEOUT) .subscribe(pe -> { if (receivedMessage.compareAndSet(null, pe.getData())) { receivedSpan.compareAndSet(null, Span.current()); @@ -213,7 +236,7 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { .build()) .build(); - createClients(otel); + createClients(otel, null); CountDownLatch latch = new CountDownLatch(2); customSpanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); @@ -344,8 +367,10 @@ public void sendBuffered() throws InterruptedException { assertEquals(2, received.size()); } - @Test - public void syncReceive() { + @ParameterizedTest + @ValueSource(strings = {"true", "false"}) + public void syncReceive(String v2) { + createClients(null, v2); StepVerifier.create(producer.createBatch(new CreateBatchOptions().setPartitionId(PARTITION_ID)) .map(b -> { b.tryAdd(new EventData(CONTENTS_BYTES)); @@ -404,11 +429,13 @@ public void syncReceiveTimeout() { assertSyncConsumerSpan(received.get(0), receivedMessages); } - @Test - public void sendAndProcess() throws InterruptedException { + @ParameterizedTest + @ValueSource(strings = {"true", "false"}) + public void sendAndProcess(String v2) throws InterruptedException { AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference receivedMessage = new AtomicReference<>(); + createClients(null, v2); CountDownLatch latch = new CountDownLatch(2); spanProcessor.notifyIfCondition(latch, span -> span == currentInProcess.get() || hasOperationName(span, SEND)); @@ -421,6 +448,7 @@ public void sendAndProcess() throws InterruptedException { .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) .initialPartitionEventPosition(Collections.singletonMap(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime))) .consumerGroup("$Default") + .configuration(createConfiguration(v2)) .checkpointStore(new SampleCheckpointStore()) .processEvent(ec -> { if (currentInProcess.compareAndSet(null, Span.current())) { @@ -528,13 +556,16 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { } } - @Test - public void sendAndProcessBatch() throws InterruptedException { + @ParameterizedTest + @ValueSource(strings = {"true", "false"}) + public void sendAndProcessBatch(String v2) throws InterruptedException { EventData message1 = new EventData(CONTENTS_BYTES); EventData message2 = new EventData(CONTENTS_BYTES); AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference> received = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(1); + + createClients(null, v2); spanProcessor.notifyIfCondition(latch, span -> span == currentInProcess.get()); StepVerifier.create(producer.send(Arrays.asList(message1, message2), new SendOptions().setPartitionId(PARTITION_ID))) .expectComplete() @@ -546,6 +577,7 @@ public void sendAndProcessBatch() throws InterruptedException { .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) .initialPartitionEventPosition(Collections.singletonMap(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime))) .consumerGroup("$Default") + .configuration(createConfiguration(v2)) .checkpointStore(new SampleCheckpointStore()) .processEventBatch(eb -> { if (currentInProcess.compareAndSet(null, Span.current())) { @@ -760,4 +792,5 @@ private List findSpans(List spans, OperationName ope private boolean hasOperationName(ReadableSpan span, OperationName operationName) { return operationName.toString().equals(span.getAttribute(OPERATION_NAME_ATTRIBUTE)); } + } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentationTests.java index b0c2d7049f814..bbe4ff19a059c 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentationTests.java @@ -30,6 +30,7 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.data.LinkData; @@ -69,9 +70,12 @@ import static com.azure.messaging.eventhubs.TestUtils.assertSpanStatus; import static com.azure.messaging.eventhubs.TestUtils.attributesToMap; import static com.azure.messaging.eventhubs.TestUtils.getSpanName; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_EVENT_HUB_PROPERTIES; +import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.GET_PARTITION_PROPERTIES; import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.PROCESS; import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.RECEIVE; import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.CHECKPOINT; +import static io.opentelemetry.api.trace.SpanKind.CLIENT; import static io.opentelemetry.api.trace.SpanKind.CONSUMER; import static io.opentelemetry.api.trace.SpanKind.INTERNAL; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -344,6 +348,46 @@ public void syncReceiveBatchEvent(boolean cancel, Throwable error, String expect } } + @Test + @SuppressWarnings("try") + public void instrumentGenericOperation() { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + String partitionId = "1"; + + StepVerifier.create(instrumentation.instrumentMono(Mono.just("partition properties"), GET_PARTITION_PROPERTIES, partitionId)) + .expectNextCount(1) + .expectComplete() + .verify(); + + assertOperationDuration(GET_PARTITION_PROPERTIES, partitionId, null); + assertGenericOperationSpan(GET_PARTITION_PROPERTIES, CLIENT, partitionId, null, null); + } + + @ParameterizedTest + @MethodSource("genericErrors") + @SuppressWarnings("try") + public void instrumentGenericOperationErrors(boolean cancel, Throwable error, String expectedErrorType, String spanDescription) { + EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, + FQDN, ENTITY_NAME, CONSUMER_GROUP, false); + + Mono operation = Mono.defer(() -> error == null ? Mono.just("eh properties") : Mono.error(error)); + StepVerifier.FirstStep stepVerifier = + StepVerifier.create(instrumentation.instrumentMono(operation, GET_EVENT_HUB_PROPERTIES, null)); + + if (cancel) { + stepVerifier.thenCancel().verify(); + } else if (error != null) { + stepVerifier.expectErrorMessage(error.getMessage()).verify(); + } else { + stepVerifier.expectNextCount(1).expectComplete().verify(); + } + + assertOperationDuration(GET_EVENT_HUB_PROPERTIES, null, expectedErrorType); + assertGenericOperationSpan(GET_EVENT_HUB_PROPERTIES, CLIENT, null, expectedErrorType, spanDescription); + } + public static Stream processErrors() { AmqpException amqpException = new AmqpException(false, AmqpErrorCondition.SERVER_BUSY_ERROR, null, new RuntimeException("test"), null); return Stream.of( @@ -414,7 +458,7 @@ public void processBatch(Throwable error, String expectedErrorType, String spanD } } - public static Stream checkpointErrors() { + public static Stream genericErrors() { return Stream.of( Arguments.of(false, null, null, null), Arguments.of(true, null, "cancelled", "cancelled"), @@ -435,7 +479,7 @@ public void checkpointWithDisabledInstrumentation() { } @ParameterizedTest - @MethodSource("checkpointErrors") + @MethodSource("genericErrors") @SuppressWarnings("try") public void checkpoint(boolean cancel, Throwable error, String expectedErrorType, String spanDescription) { EventHubsConsumerInstrumentation instrumentation = new EventHubsConsumerInstrumentation(tracer, meter, @@ -540,36 +584,27 @@ private static EventData createEventData(Instant enqueuedTime, String traceparen } private SpanData assertReceiveSpan(int expectedBatchSize, String partitionId, String expectedErrorType, String spanDescription) { - assertEquals(1, spanProcessor.getEndedSpans().size()); - SpanData span = spanProcessor.getEndedSpans().get(0).toSpanData(); - assertEquals(getSpanName(RECEIVE, ENTITY_NAME), span.getName()); - assertEquals(CONSUMER, span.getKind()); + SpanData span = assertGenericOperationSpan(RECEIVE, CLIENT, partitionId, expectedErrorType, spanDescription); Map attributes = attributesToMap(span.getAttributes()); - assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, RECEIVE, attributes); - assertSpanStatus(spanDescription, span); - assertEquals((long) expectedBatchSize, attributes.get("messaging.batch.message_count")); return span; } private SpanData assertProcessSpan(String partitionId, String expectedErrorType, String spanDescription) { - assertEquals(1, spanProcessor.getEndedSpans().size()); - SpanData span = spanProcessor.getEndedSpans().get(0).toSpanData(); - assertEquals(getSpanName(PROCESS, ENTITY_NAME), span.getName()); - assertEquals(CONSUMER, span.getKind()); - Map attributes = attributesToMap(span.getAttributes()); - assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, PROCESS, attributes); - assertSpanStatus(spanDescription, span); - return span; + return assertGenericOperationSpan(PROCESS, CONSUMER, partitionId, expectedErrorType, spanDescription); } private SpanData assertCheckpointSpan(String partitionId, String expectedErrorType, String spanDescription) { + return assertGenericOperationSpan(CHECKPOINT, INTERNAL, partitionId, expectedErrorType, spanDescription); + } + + private SpanData assertGenericOperationSpan(OperationName operation, SpanKind kind, String partitionId, String expectedErrorType, String spanDescription) { assertEquals(1, spanProcessor.getEndedSpans().size()); SpanData span = spanProcessor.getEndedSpans().get(0).toSpanData(); - assertEquals(getSpanName(CHECKPOINT, ENTITY_NAME), span.getName()); - assertEquals(INTERNAL, span.getKind()); + assertEquals(getSpanName(operation, ENTITY_NAME), span.getName()); + assertEquals(kind, span.getKind()); Map attributes = attributesToMap(span.getAttributes()); - assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, CHECKPOINT, attributes); + assertAllAttributes(FQDN, ENTITY_NAME, partitionId, CONSUMER_GROUP, expectedErrorType, operation, attributes); assertSpanStatus(spanDescription, span); return span; } From 839d1211c8d792baf5e93a1b0811aa170e7420e6 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 10:59:17 -0700 Subject: [PATCH 03/21] fix tests and update --- .../SynchronousPartitionReceiver.java | 12 +- .../eventhubs/SynchronousReceiver.java | 139 ------------------ .../eventhubs/TracingIntegrationTests.java | 12 +- 3 files changed, 10 insertions(+), 153 deletions(-) delete mode 100644 sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousPartitionReceiver.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousPartitionReceiver.java index 608ecb4fa8ea5..4f0b66715f96c 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousPartitionReceiver.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousPartitionReceiver.java @@ -5,16 +5,14 @@ import com.azure.core.amqp.implementation.WindowedSubscriber; import com.azure.core.amqp.implementation.WindowedSubscriber.WindowedSubscriberOptions; -import com.azure.core.util.Context; import com.azure.core.util.IterableStream; -import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsTracer; +import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; import com.azure.messaging.eventhubs.models.EventPosition; import com.azure.messaging.eventhubs.models.PartitionEvent; import com.azure.messaging.eventhubs.models.ReceiveOptions; import reactor.core.publisher.Flux; import java.time.Duration; -import java.time.Instant; import java.util.Collections; import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; @@ -26,8 +24,7 @@ */ final class SynchronousPartitionReceiver { private static final String TERMINAL_MESSAGE = "The receiver client is terminated. Re-create the client to continue receive attempt."; - private static final String SYNC_RECEIVE_SPAN_NAME = "EventHubs.receiveFromPartition"; - private final EventHubsTracer tracer; + private final EventHubsConsumerInstrumentation instrumentation; private final AtomicReference receiver = new AtomicReference<>(null); /** @@ -38,7 +35,7 @@ final class SynchronousPartitionReceiver { SynchronousPartitionReceiver(EventHubConsumerAsyncClient client) { Objects.requireNonNull(client, "'client' cannot be null."); this.receiver.set(new DelegatingReceiver(client)); - this.tracer = client.getInstrumentation().getTracer(); + this.instrumentation = client.getInstrumentation(); } /** @@ -88,8 +85,7 @@ private WindowedSubscriber createSubscriber(String partitionId) final WindowedSubscriberOptions options = new WindowedSubscriberOptions<>(); options.setWindowDecorator(toDecorate -> { // Decorates the provided 'toDecorate' flux for tracing the signals (events, termination) it produces. - final Instant startTime = tracer.isEnabled() ? Instant.now() : null; - return tracer.reportSyncReceiveSpan(SYNC_RECEIVE_SPAN_NAME, startTime, toDecorate, Context.NONE); + return instrumentation.syncReceive(toDecorate, partitionId); }); return new WindowedSubscriber<>(Collections.singletonMap(PARTITION_ID_KEY, partitionId), TERMINAL_MESSAGE, options); } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java deleted file mode 100644 index 6f013e76ac5a6..0000000000000 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/SynchronousReceiver.java +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.messaging.eventhubs; - -import com.azure.core.amqp.implementation.WindowedSubscriber; -import com.azure.core.amqp.implementation.WindowedSubscriber.WindowedSubscriberOptions; -import com.azure.core.util.IterableStream; -import com.azure.core.util.logging.ClientLogger; -import com.azure.messaging.eventhubs.implementation.instrumentation.EventHubsConsumerInstrumentation; -import com.azure.messaging.eventhubs.models.EventPosition; -import com.azure.messaging.eventhubs.models.PartitionEvent; -import com.azure.messaging.eventhubs.models.ReceiveOptions; -import reactor.core.publisher.Flux; - -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicReference; - -import static com.azure.messaging.eventhubs.implementation.ClientConstants.PARTITION_ID_KEY; - -/** - * A type that channels synchronous receive requests to a backing asynchronous receiver client. - */ -final class SynchronousReceiver { - private static final String TERMINAL_MESSAGE; - private static final WindowedSubscriber DISPOSED; - static { - TERMINAL_MESSAGE = "The receiver client is terminated. Re-create the client to continue receive attempt."; - DISPOSED = Flux.error(new RuntimeException("Disposed.")) - .subscribeWith(new WindowedSubscriber<>(new HashMap<>(0), TERMINAL_MESSAGE, new WindowedSubscriberOptions<>())); - } - private static final String SYNC_RECEIVE_SPAN_NAME = "EventHubs.receiveFromPartition"; - private final ClientLogger logger; - private final EventHubConsumerAsyncClient asyncClient; - private final EventHubsConsumerInstrumentation instrumentation; - private final AtomicReference> subscriber = new AtomicReference<>(null); - - /** - * Creates a SynchronousReceiver. - * - * @param logger the logger to use. - * @param asyncClient the backing asynchronous client to connect to the broker, delegate message requesting and receive. - */ - SynchronousReceiver(ClientLogger logger, EventHubConsumerAsyncClient asyncClient) { - this.logger = Objects.requireNonNull(logger, "'logger' cannot be null."); - this.asyncClient = Objects.requireNonNull(asyncClient, "'asyncClient' cannot be null."); - this.instrumentation = asyncClient.getInstrumentation(); - } - - /** - * Request a specified number of event and obtain an {@link IterableStream} streaming the received event. - * - * @param partitionId Identifier of the partition to read events from. - * @param startingPosition Position within the Event Hub partition to begin consuming events. - * @param receiveOptions Options when receiving events from the partition. - * @param maxEvents the maximum number of event to receive. - * @param maxWaitTime the upper bound for the time to wait to receive the requested number of event. - * - * @return an {@link IterableStream} of at most {@code maxEvents} event. - */ - IterableStream receive(String partitionId, EventPosition startingPosition, - ReceiveOptions receiveOptions, int maxEvents, Duration maxWaitTime) { - Objects.requireNonNull(partitionId, "'partitionId' cannot be null."); - Objects.requireNonNull(startingPosition, "'startingPosition' cannot be null."); - Objects.requireNonNull(receiveOptions, "'receiveOptions' cannot be null."); - - final WindowedSubscriber s = subscriber.get(); - if (s != null) { - return s.enqueueRequest(maxEvents, maxWaitTime); - } else { - return subscribeOnce(partitionId, startingPosition, receiveOptions).enqueueRequest(maxEvents, maxWaitTime); - } - } - - /** - * Disposes the SynchronousReceiver. - *

- * Once disposed, the {@link IterableStream} for any future or pending receive requests will receive terminated error. - *

- */ - void dispose() { - final WindowedSubscriber s = subscriber.getAndSet(DISPOSED); - if (s != null) { - s.dispose(); - } - } - - /** - * Obtain a {@link WindowedSubscriber} that is subscribed to the asynchronous message stream produced by the backing - * asynchronous client. - *

- * The method subscribes only once and cache the subscriber. - *

- *

- * The subscriber exposes synchronous receive API and channels receive requests to the backing asynchronous message - * stream. - *

- * - * @param partitionId Identifier of the partition to read events from. - * @param startingPosition Position within the Event Hub partition to begin consuming events. - * @param receiveOptions Options when receiving events from the partition. - * - * @return the subscriber to channel synchronous receive requests to the upstream. - */ - private WindowedSubscriber subscribeOnce(String partitionId, EventPosition startingPosition, - ReceiveOptions receiveOptions) { - if (!asyncClient.isV2()) { - throw logger.logExceptionAsError(new UnsupportedOperationException("SynchronousReceiver requires v2 mode.")); - } - final WindowedSubscriber s = createSubscriber(partitionId); - if (subscriber.compareAndSet(null, s)) { - // In case of concurrent invocation, the 's' created by the thread which lost the CAS race is eligible for GC. - // There is no leak, as 's' is in mere constructed state and don’t own any leak-able resource yet. - final Flux upstream = asyncClient.receiveFromPartition(partitionId, startingPosition, receiveOptions); - upstream.subscribeWith(s); - } - return subscriber.get(); - } - - /** - * Create a {@link WindowedSubscriber} capable of bridging synchronous receive requests to an upstream of - * asynchronous event. - * - * @param partitionId Identifier of the partition to read events from. - * - * @return The subscriber. - */ - private WindowedSubscriber createSubscriber(String partitionId) { - final WindowedSubscriberOptions options = new WindowedSubscriberOptions<>(); - options.setWindowDecorator(toDecorate -> { - // Decorates the provided 'toDecorate' flux for tracing the signals (events, termination) it produces. - return instrumentation.syncReceive(toDecorate, partitionId); - }); - return new WindowedSubscriber<>(Collections.singletonMap(PARTITION_ID_KEY, partitionId), TERMINAL_MESSAGE, options); - } -} diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index 021d96ecf438b..c8cc07899f4b9 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -175,7 +175,7 @@ public void sendAndReceiveFromPartition(String v2) throws InterruptedException { .expectComplete() .verify(DEFAULT_TIMEOUT); - assertTrue(latch.await(30, TimeUnit.SECONDS)); + assertTrue(latch.await(10, TimeUnit.SECONDS)); List spans = spanProcessor.getEndedSpans(); @@ -389,7 +389,7 @@ public void syncReceive(String v2) { assertEquals(0, findSpans(spans, PROCESS).size()); List received = findSpans(spans, RECEIVE); - assertSyncConsumerSpan(received.get(0), receivedMessages); + assertSyncReceiveSpan(received.get(0), receivedMessages); } @Test @@ -413,7 +413,7 @@ public void syncReceiveWithOptions() { assertEquals(0, findSpans(spans, PROCESS).size()); List received = findSpans(spans, RECEIVE); - assertSyncConsumerSpan(received.get(0), receivedMessages); + assertSyncReceiveSpan(received.get(0), receivedMessages); } @Test @@ -426,7 +426,7 @@ public void syncReceiveTimeout() { assertEquals(0, findSpans(spans, PROCESS).size()); List received = findSpans(spans, RECEIVE); - assertSyncConsumerSpan(received.get(0), receivedMessages); + assertSyncReceiveSpan(received.get(0), receivedMessages); } @ParameterizedTest @@ -698,8 +698,8 @@ private void assertSendSpan(ReadableSpan actual, List messages) { } } - private void assertSyncConsumerSpan(ReadableSpan actual, List messages) { - assertEquals(SpanKind.CONSUMER, actual.getKind()); + private void assertSyncReceiveSpan(ReadableSpan actual, List messages) { + assertEquals(SpanKind.CLIENT, actual.getKind()); assertEquals(StatusCode.UNSET, actual.toSpanData().getStatus().getStatusCode()); List links = actual.toSpanData().getLinks(); assertEquals("receive", actual.getAttribute(OPERATION_NAME_ATTRIBUTE)); From 9ec9620ea0292c9331146680a6a8a97e78e4aeeb Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 11:48:24 -0700 Subject: [PATCH 04/21] test: remove v2 stack --- .../eventhubs/TracingIntegrationTests.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index c8cc07899f4b9..c79b69f960b3d 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -152,7 +152,7 @@ protected void afterTest() { } @ParameterizedTest - @ValueSource(strings = {"true", "false"}) + @ValueSource(strings = {"false"}) @NullSource public void sendAndReceiveFromPartition(String v2) throws InterruptedException { AtomicReference receivedMessage = new AtomicReference<>(); @@ -175,7 +175,7 @@ public void sendAndReceiveFromPartition(String v2) throws InterruptedException { .expectComplete() .verify(DEFAULT_TIMEOUT); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); List spans = spanProcessor.getEndedSpans(); @@ -210,7 +210,7 @@ public void sendAndReceive() throws InterruptedException { .expectComplete() .verify(DEFAULT_TIMEOUT); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); List spans = spanProcessor.getEndedSpans(); @@ -352,7 +352,7 @@ public void sendBuffered() throws InterruptedException { .expectComplete() .verify(Duration.ofSeconds(60)); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); List spans = spanProcessor.getEndedSpans(); List message = findSpans(spans, EVENT); @@ -368,7 +368,7 @@ public void sendBuffered() throws InterruptedException { } @ParameterizedTest - @ValueSource(strings = {"true", "false"}) + @ValueSource(strings = {"false"}) public void syncReceive(String v2) { createClients(null, v2); StepVerifier.create(producer.createBatch(new CreateBatchOptions().setPartitionId(PARTITION_ID)) @@ -430,7 +430,7 @@ public void syncReceiveTimeout() { } @ParameterizedTest - @ValueSource(strings = {"true", "false"}) + @ValueSource(strings = {"false"}) public void sendAndProcess(String v2) throws InterruptedException { AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference receivedMessage = new AtomicReference<>(); @@ -461,7 +461,7 @@ public void sendAndProcess(String v2) throws InterruptedException { toClose((Closeable) () -> processor.stop()); processor.start(); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); processor.stop(); assertTrue(currentInProcess.get().getSpanContext().isValid()); @@ -531,7 +531,7 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { toClose((Closeable) () -> processor.stop()); processor.start(); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); processor.stop(); } @@ -557,7 +557,7 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { } @ParameterizedTest - @ValueSource(strings = {"true", "false"}) + @ValueSource(strings = {"false"}) public void sendAndProcessBatch(String v2) throws InterruptedException { EventData message1 = new EventData(CONTENTS_BYTES); EventData message2 = new EventData(CONTENTS_BYTES); @@ -595,7 +595,7 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { .buildEventProcessorClient(); toClose((Closeable) () -> processor.stop()); processor.start(); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); processor.stop(); List spans = spanProcessor.getEndedSpans(); @@ -651,7 +651,7 @@ public void sendProcessAndFail() throws InterruptedException { toClose((Closeable) () -> processor.stop()); processor.start(); - assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(30, TimeUnit.SECONDS)); processor.stop(); List spans = spanProcessor.getEndedSpans(); From 9c2c9478b074bf472a8851f6dd9399b8cfb5a1eb Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 13:43:39 -0700 Subject: [PATCH 05/21] tryme --- .../eventhubs/MessageFluxWrapper.java | 2 +- .../eventhubs/IntegrationTestBase.java | 2 +- .../messaging/eventhubs/TestSpanProcessor.java | 12 +++++++----- .../eventhubs/TracingIntegrationTests.java | 18 ++++++++++-------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java index cc26669c98794..48500cf8b9143 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java @@ -36,7 +36,7 @@ boolean isTerminated() { void cancel() { if (!isV2) { - receiveLinkProcessor.cancel(); + receiveLinkProcessor.dispose(); } } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java index 29072d6a6fe4e..9a74270a3328e 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java @@ -54,7 +54,7 @@ public abstract class IntegrationTestBase extends TestBase { // This is a good idea to do in any production application as well - no point in waiting too long protected static final AmqpRetryOptions RETRY_OPTIONS = new AmqpRetryOptions() .setTryTimeout(Duration.ofSeconds(3)) - .setMaxDelay(Duration.ofSeconds(3)) + .setMaxDelay(Duration.ofSeconds(1)) .setMaxRetries(10); protected final ClientLogger logger; diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java index 0944d36bb7216..591f2f913da4f 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java @@ -6,6 +6,7 @@ import com.azure.core.util.logging.ClientLogger; import com.azure.core.util.logging.LoggingEventBuilder; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.context.Context; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.common.InstrumentationScopeInfo; @@ -56,6 +57,7 @@ public boolean isStartRequired() { @Override public void onEnd(ReadableSpan readableSpan) { SpanData span = readableSpan.toSpanData(); + Attributes attributes = span.getAttributes(); InstrumentationScopeInfo instrumentationScopeInfo = span.getInstrumentationScopeInfo(); LoggingEventBuilder log = LOGGER.atInfo() @@ -67,7 +69,7 @@ public void onEnd(ReadableSpan readableSpan) { .addKeyValue("kind", span.getKind()) .addKeyValue("tracerName", instrumentationScopeInfo.getName()) .addKeyValue("tracerVersion", instrumentationScopeInfo.getVersion()) - .addKeyValue("attributes", span.getAttributes()); + .addKeyValue("attributes", attributes); for (int i = 0; i < span.getLinks().size(); i++) { LinkData link = span.getLinks().get(i); @@ -86,10 +88,10 @@ public void onEnd(ReadableSpan readableSpan) { // Various attribute keys can be found in: // sdk/core/azure-core-metrics-opentelemetry/src/main/java/com/azure/core/metrics/opentelemetry/OpenTelemetryAttributes.java // sdk/core/azure-core-tracing-opentelemetry/src/main/java/com/azure/core/tracing/opentelemetry/OpenTelemetryUtils.java - assertEquals("Microsoft.EventHub", readableSpan.getAttribute(AttributeKey.stringKey("az.namespace"))); - assertEquals("eventhubs", readableSpan.getAttribute(AttributeKey.stringKey("messaging.system"))); - assertEquals(entityName, readableSpan.getAttribute(AttributeKey.stringKey("messaging.destination.name"))); - assertEquals(namespace, readableSpan.getAttribute(AttributeKey.stringKey("server.address"))); + assertEquals("Microsoft.EventHub", attributes.get(AttributeKey.stringKey("az.namespace"))); + assertEquals("eventhubs", attributes.get(AttributeKey.stringKey("messaging.system"))); + assertEquals(entityName, attributes.get(AttributeKey.stringKey("messaging.destination.name"))); + assertEquals(namespace, attributes.get(AttributeKey.stringKey("server.address"))); } public void notifyIfCondition(CountDownLatch countDownLatch, Predicate filter) { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index c79b69f960b3d..56fef4ce52d2a 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -116,6 +116,7 @@ private Configuration createConfiguration(String v2Stack) { return configBuilder .putProperty("com.azure.messaging.eventhubs.v2", v2Stack) + .putProperty("com.azure.core.amqp.cache", v2Stack) .build(); } @@ -152,7 +153,7 @@ protected void afterTest() { } @ParameterizedTest - @ValueSource(strings = {"false"}) + @ValueSource(strings = {"true", "false"}) @NullSource public void sendAndReceiveFromPartition(String v2) throws InterruptedException { AtomicReference receivedMessage = new AtomicReference<>(); @@ -272,6 +273,11 @@ public void sendAndReceiveParallel() throws InterruptedException { int messageCount = 5; CountDownLatch latch = new CountDownLatch(messageCount); spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS)); + + StepVerifier.create(producer.send(data, new SendOptions())) + .expectComplete() + .verify(DEFAULT_TIMEOUT); + StepVerifier.create(consumer .receive() .take(messageCount) @@ -291,10 +297,6 @@ public void sendAndReceiveParallel() throws InterruptedException { .expectComplete() .verify(DEFAULT_TIMEOUT); - StepVerifier.create(producer.send(data, new SendOptions())) - .expectComplete() - .verify(DEFAULT_TIMEOUT); - assertTrue(latch.await(20, TimeUnit.SECONDS)); List spans = spanProcessor.getEndedSpans(); List received = findSpans(spans, PROCESS); @@ -368,7 +370,7 @@ public void sendBuffered() throws InterruptedException { } @ParameterizedTest - @ValueSource(strings = {"false"}) + @ValueSource(strings = {"true", "false"}) public void syncReceive(String v2) { createClients(null, v2); StepVerifier.create(producer.createBatch(new CreateBatchOptions().setPartitionId(PARTITION_ID)) @@ -430,7 +432,7 @@ public void syncReceiveTimeout() { } @ParameterizedTest - @ValueSource(strings = {"false"}) + @ValueSource(strings = {"true", "false"}) public void sendAndProcess(String v2) throws InterruptedException { AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference receivedMessage = new AtomicReference<>(); @@ -557,7 +559,7 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { } @ParameterizedTest - @ValueSource(strings = {"false"}) + @ValueSource(strings = {"true", "false"}) public void sendAndProcessBatch(String v2) throws InterruptedException { EventData message1 = new EventData(CONTENTS_BYTES); EventData message2 = new EventData(CONTENTS_BYTES); From 6387ea523800992878e9acfa2a0bd61129af02b9 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 15:42:09 -0700 Subject: [PATCH 06/21] tryme --- .../java/com/azure/messaging/eventhubs/V2StackSupport.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/V2StackSupport.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/V2StackSupport.java index 372bef8ac8736..25a2dfc757db7 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/V2StackSupport.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/V2StackSupport.java @@ -100,8 +100,8 @@ private boolean isOptedOut(Configuration configuration, ConfigurationProperty Date: Thu, 12 Sep 2024 17:48:13 -0700 Subject: [PATCH 07/21] up --- .../com/azure/messaging/eventhubs/MessageFluxWrapper.java | 7 ++++--- .../java/com/azure/messaging/eventhubs/V2StackSupport.java | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java index 48500cf8b9143..edab872b8ed75 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java @@ -3,6 +3,7 @@ package com.azure.messaging.eventhubs; +import com.azure.core.amqp.implementation.MessageFlux; import com.azure.messaging.eventhubs.implementation.AmqpReceiveLinkProcessor; import org.apache.qpid.proton.message.Message; import reactor.core.publisher.Flux; @@ -11,10 +12,10 @@ final class MessageFluxWrapper { private final AmqpReceiveLinkProcessor receiveLinkProcessor; - private final Flux messageFlux; + private final MessageFlux messageFlux; private final boolean isV2; - MessageFluxWrapper(Flux messageFlux) { + MessageFluxWrapper(MessageFlux messageFlux) { this.messageFlux = Objects.requireNonNull(messageFlux, "'messageFlux' cannot be null."); this.receiveLinkProcessor = null; this.isV2 = true; @@ -36,7 +37,7 @@ boolean isTerminated() { void cancel() { if (!isV2) { - receiveLinkProcessor.dispose(); + receiveLinkProcessor.cancel(); } } } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/V2StackSupport.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/V2StackSupport.java index 25a2dfc757db7..372bef8ac8736 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/V2StackSupport.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/V2StackSupport.java @@ -100,8 +100,8 @@ private boolean isOptedOut(Configuration configuration, ConfigurationProperty Date: Thu, 12 Sep 2024 17:51:30 -0700 Subject: [PATCH 08/21] up --- .../com/azure/messaging/eventhubs/MessageFluxWrapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java index edab872b8ed75..450e68212eefc 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java @@ -12,10 +12,10 @@ final class MessageFluxWrapper { private final AmqpReceiveLinkProcessor receiveLinkProcessor; - private final MessageFlux messageFlux; + private final Flux messageFlux; private final boolean isV2; - MessageFluxWrapper(MessageFlux messageFlux) { + MessageFluxWrapper(Flux messageFlux) { this.messageFlux = Objects.requireNonNull(messageFlux, "'messageFlux' cannot be null."); this.receiveLinkProcessor = null; this.isV2 = true; From 9304617def99499883c5b244d7d72296b3927268 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 18:45:24 -0700 Subject: [PATCH 09/21] up --- .../azure/messaging/eventhubs/TestSpanProcessor.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java index 591f2f913da4f..0fd11d0b3d4dd 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TestSpanProcessor.java @@ -88,10 +88,12 @@ public void onEnd(ReadableSpan readableSpan) { // Various attribute keys can be found in: // sdk/core/azure-core-metrics-opentelemetry/src/main/java/com/azure/core/metrics/opentelemetry/OpenTelemetryAttributes.java // sdk/core/azure-core-tracing-opentelemetry/src/main/java/com/azure/core/tracing/opentelemetry/OpenTelemetryUtils.java - assertEquals("Microsoft.EventHub", attributes.get(AttributeKey.stringKey("az.namespace"))); - assertEquals("eventhubs", attributes.get(AttributeKey.stringKey("messaging.system"))); - assertEquals(entityName, attributes.get(AttributeKey.stringKey("messaging.destination.name"))); - assertEquals(namespace, attributes.get(AttributeKey.stringKey("server.address"))); + String messagingSystem = attributes.get(AttributeKey.stringKey("messaging.system")); + if ("eventhubs".equals(messagingSystem)) { + assertEquals("Microsoft.EventHub", attributes.get(AttributeKey.stringKey("az.namespace"))); + assertEquals(entityName, attributes.get(AttributeKey.stringKey("messaging.destination.name"))); + assertEquals(namespace, attributes.get(AttributeKey.stringKey("server.address"))); + } } public void notifyIfCondition(CountDownLatch countDownLatch, Predicate filter) { From 379a6aef88e19d31c74b888ec97b7a072f7146fe Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 19:59:05 -0700 Subject: [PATCH 10/21] up --- .../eventhubs/TracingIntegrationTests.java | 98 ++++++++++--------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index 56fef4ce52d2a..bae947c0a0886 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -12,6 +12,7 @@ import com.azure.core.util.logging.ClientLogger; import com.azure.messaging.eventhubs.implementation.instrumentation.OperationName; import com.azure.messaging.eventhubs.models.CreateBatchOptions; +import com.azure.messaging.eventhubs.models.EventContext; import com.azure.messaging.eventhubs.models.EventPosition; import com.azure.messaging.eventhubs.models.PartitionEvent; import com.azure.messaging.eventhubs.models.ReceiveOptions; @@ -91,18 +92,6 @@ public TracingIntegrationTests() { @Override protected void beforeTest() { - GlobalOpenTelemetry.resetForTest(); - - spanProcessor = toClose(new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), testName)); - OpenTelemetrySdk otel = OpenTelemetrySdk.builder() - .setTracerProvider( - SdkTracerProvider.builder() - .addSpanProcessor(spanProcessor) - .build()) - .buildAndRegisterGlobal(); - - createClients(otel, null); - testStartTime = Instant.now().minusSeconds(1); data = new EventData(CONTENTS_BYTES); } @@ -121,12 +110,21 @@ private Configuration createConfiguration(String v2Stack) { } private void createClients(OpenTelemetrySdk otel, String v2Stack) { + GlobalOpenTelemetry.resetForTest(); dispose(); + + spanProcessor = toClose(new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), testName)); + ClientOptions options = new ClientOptions(); - if (otel != null) { - options.setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); + if (otel == null) { + otel = OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .addSpanProcessor(spanProcessor) + .build()) + .buildAndRegisterGlobal(); } - + options.setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); Configuration config = createConfiguration(v2Stack); producer = toClose(createBuilder() @@ -156,18 +154,18 @@ protected void afterTest() { @ValueSource(strings = {"true", "false"}) @NullSource public void sendAndReceiveFromPartition(String v2) throws InterruptedException { - AtomicReference receivedMessage = new AtomicReference<>(); - AtomicReference receivedSpan = new AtomicReference<>(); - createClients(null, v2); + AtomicReference receivedMessage = new AtomicReference<>(); + AtomicReference receivedSpan = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(2); spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer .receiveFromPartition(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime)) .take(1) .subscribe(pe -> { - if (receivedMessage.compareAndSet(null, pe.getData())) { + if (receivedMessage.compareAndSet(null, pe)) { receivedSpan.compareAndSet(null, Span.current()); } })); @@ -187,13 +185,15 @@ public void sendAndReceiveFromPartition(String v2) throws InterruptedException { List received = findSpans(spans, PROCESS).stream() .filter(s -> s == receivedSpan.get()).collect(toList()); - assertConsumerSpan(received.get(0), receivedMessage.get()); + assertConsumerSpan(received.get(0), receivedMessage.get().getData(), receivedMessage.get().getPartitionContext().getPartitionId()); assertNull(received.get(0).getAttribute(AttributeKey.stringKey("messaging.consumer.group.name"))); } @Test public void sendAndReceive() throws InterruptedException { - AtomicReference receivedMessage = new AtomicReference<>(); + createClients(null, null); + + AtomicReference receivedMessage = new AtomicReference<>(); AtomicReference receivedSpan = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(2); @@ -202,7 +202,7 @@ public void sendAndReceive() throws InterruptedException { .receive() .take(DEFAULT_TIMEOUT) .subscribe(pe -> { - if (receivedMessage.compareAndSet(null, pe.getData())) { + if (receivedMessage.compareAndSet(null, pe)) { receivedSpan.compareAndSet(null, Span.current()); } })); @@ -222,12 +222,13 @@ public void sendAndReceive() throws InterruptedException { List received = findSpans(spans, PROCESS).stream() .filter(s -> s == receivedSpan.get()).collect(toList()); - assertConsumerSpan(received.get(0), receivedMessage.get()); + assertConsumerSpan(received.get(0), receivedMessage.get().getData(), receivedMessage.get().getPartitionContext().getPartitionId()); } @Test public void sendAndReceiveCustomProvider() throws InterruptedException { - AtomicReference receivedMessage = new AtomicReference<>(); + + AtomicReference receivedMessage = new AtomicReference<>(); AtomicReference receivedSpan = new AtomicReference<>(); TestSpanProcessor customSpanProcessor = toClose(new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), "sendAndReceiveCustomProvider")); @@ -236,16 +237,14 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { .addSpanProcessor(customSpanProcessor) .build()) .build(); - createClients(otel, null); - CountDownLatch latch = new CountDownLatch(2); customSpanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer.receive() .take(1) .subscribe(pe -> { - if (receivedMessage.compareAndSet(null, pe.getData())) { + if (receivedMessage.compareAndSet(null, pe)) { receivedSpan.compareAndSet(null, Span.current()); } })); @@ -265,13 +264,16 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { List received = findSpans(spans, PROCESS).stream() .filter(s -> s == receivedSpan.get()).collect(toList()); - assertConsumerSpan(received.get(0), receivedMessage.get()); + assertConsumerSpan(received.get(0), receivedMessage.get().getData(), receivedMessage.get().getPartitionContext().getPartitionId()); } @Test public void sendAndReceiveParallel() throws InterruptedException { + createClients(null, null); + int messageCount = 5; CountDownLatch latch = new CountDownLatch(messageCount); + spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS)); StepVerifier.create(producer.send(data, new SendOptions())) @@ -305,6 +307,8 @@ public void sendAndReceiveParallel() throws InterruptedException { @Test public void sendBuffered() throws InterruptedException { + createClients(null, null); + CountDownLatch latch = new CountDownLatch(3); spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS) || hasOperationName(span, SEND)); @@ -396,6 +400,7 @@ public void syncReceive(String v2) { @Test public void syncReceiveWithOptions() { + createClients(null, null); StepVerifier.create(producer.createBatch(new CreateBatchOptions().setPartitionId(PARTITION_ID)) .map(b -> { b.tryAdd(new EventData(CONTENTS_BYTES)); @@ -420,6 +425,7 @@ public void syncReceiveWithOptions() { @Test public void syncReceiveTimeout() { + createClients(null, null); List receivedMessages = consumerSync.receiveFromPartition(PARTITION_ID, 2, EventPosition.fromEnqueuedTime(testStartTime), Duration.ofSeconds(1)) .stream().collect(toList()); @@ -434,10 +440,11 @@ public void syncReceiveTimeout() { @ParameterizedTest @ValueSource(strings = {"true", "false"}) public void sendAndProcess(String v2) throws InterruptedException { + createClients(null, v2); + AtomicReference currentInProcess = new AtomicReference<>(); - AtomicReference receivedMessage = new AtomicReference<>(); + AtomicReference receivedMessage = new AtomicReference<>(); - createClients(null, v2); CountDownLatch latch = new CountDownLatch(2); spanProcessor.notifyIfCondition(latch, span -> span == currentInProcess.get() || hasOperationName(span, SEND)); @@ -448,13 +455,13 @@ public void sendAndProcess(String v2) throws InterruptedException { EventHubClientBuilder builder = createBuilder(); processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) - .initialPartitionEventPosition(Collections.singletonMap(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime))) + .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") .configuration(createConfiguration(v2)) .checkpointStore(new SampleCheckpointStore()) .processEvent(ec -> { if (currentInProcess.compareAndSet(null, Span.current())) { - receivedMessage.compareAndSet(null, ec.getEventData()); + receivedMessage.compareAndSet(null, ec); } ec.updateCheckpoint(); }) @@ -478,7 +485,7 @@ public void sendAndProcess(String v2) throws InterruptedException { List processed = findSpans(spans, PROCESS) .stream().filter(p -> p == currentInProcess.get()).collect(toList()); assertEquals(1, processed.size()); - assertConsumerSpan(processed.get(0), receivedMessage.get()); + assertConsumerSpan(processed.get(0), receivedMessage.get().getEventData(), receivedMessage.get().getPartitionContext().getPartitionId()); assertNull(processed.get(0).getAttribute(AttributeKey.stringKey("messaging.consumer.group.name"))); SpanContext parentSpanContext = currentInProcess.get().getSpanContext(); @@ -492,14 +499,14 @@ public void sendAndProcess(String v2) throws InterruptedException { @Test @SuppressWarnings("try") public void sendNotInstrumentedAndProcess() throws InterruptedException { - + createClients(null, null); EventHubProducerAsyncClient notInstrumentedProducer = toClose(createBuilder() .clientOptions(new ClientOptions().setTracingOptions(new TracingOptions().setEnabled(false))) .buildAsyncProducerClient()); EventData message1 = new EventData(CONTENTS_BYTES); EventData message2 = new EventData(CONTENTS_BYTES); - List received = new ArrayList<>(); + List received = new ArrayList<>(); CountDownLatch latch = new CountDownLatch(2); spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS) && !span.getParentSpanContext().isValid()); StepVerifier.create(notInstrumentedProducer.send(Arrays.asList(message1, message2), new SendOptions().setPartitionId(PARTITION_ID))) @@ -518,12 +525,12 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { try (Scope scope = test.makeCurrent()) { processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) - .initialPartitionEventPosition(Collections.singletonMap(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime))) + .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") .checkpointStore(new SampleCheckpointStore()) .processEvent(ec -> { if (!ec.getEventData().getProperties().containsKey("traceparent")) { - received.add(ec.getEventData()); + received.add(ec); } ec.updateCheckpoint(); }) @@ -543,11 +550,11 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { .filter(s -> !s.getParentSpanContext().isValid()) .collect(toList()); assertTrue(processed.size() >= 2); - assertConsumerSpan(processed.get(0), received.get(0)); + assertConsumerSpan(processed.get(0), received.get(0).getEventData(), received.get(0).getPartitionContext().getPartitionId()); List checkpointed = findSpans(spans, CHECKPOINT).stream().collect(toList()); for (int i = 1; i < processed.size(); i++) { - assertConsumerSpan(processed.get(i), received.get(i)); + assertConsumerSpan(processed.get(i), received.get(i).getEventData(), received.get(i).getPartitionContext().getPartitionId()); SpanContext parentSpanContext = processed.get(i).getSpanContext(); assertNotEquals(processed.get(0).getSpanContext().getTraceId(), parentSpanContext.getTraceId()); List checkpointedChildren = checkpointed.stream() @@ -561,13 +568,14 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { @ParameterizedTest @ValueSource(strings = {"true", "false"}) public void sendAndProcessBatch(String v2) throws InterruptedException { + createClients(null, v2); + EventData message1 = new EventData(CONTENTS_BYTES); EventData message2 = new EventData(CONTENTS_BYTES); AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference> received = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(1); - createClients(null, v2); spanProcessor.notifyIfCondition(latch, span -> span == currentInProcess.get()); StepVerifier.create(producer.send(Arrays.asList(message1, message2), new SendOptions().setPartitionId(PARTITION_ID))) .expectComplete() @@ -577,7 +585,7 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) - .initialPartitionEventPosition(Collections.singletonMap(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime))) + .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") .configuration(createConfiguration(v2)) .checkpointStore(new SampleCheckpointStore()) @@ -625,6 +633,8 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { @Test public void sendProcessAndFail() throws InterruptedException { + createClients(null, null); + AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference> received = new AtomicReference<>(); CountDownLatch latch = new CountDownLatch(2); @@ -638,7 +648,7 @@ public void sendProcessAndFail() throws InterruptedException { processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) - .initialPartitionEventPosition(Collections.singletonMap(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime))) + .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") .checkpointStore(new SampleCheckpointStore()) .processEventBatch(eb -> { @@ -719,13 +729,13 @@ private void assertSyncReceiveSpan(ReadableSpan actual, List mes } } - private void assertConsumerSpan(ReadableSpan actual, EventData message) { + private void assertConsumerSpan(ReadableSpan actual, EventData message, String partitionId) { SpanData spanData = actual.toSpanData(); assertEquals(SpanKind.CONSUMER, actual.getKind()); assertEquals(StatusCode.UNSET, spanData.getStatus().getStatusCode()); assertNotNull(actual.getAttribute(OPERATION_NAME_ATTRIBUTE)); assertNotNull(actual.getAttribute(OPERATION_TYPE_ATTRIBUTE)); - assertEquals(PARTITION_ID, actual.getAttribute(AttributeKey.stringKey("messaging.destination.partition.id"))); + assertEquals(partitionId, actual.getAttribute(AttributeKey.stringKey("messaging.destination.partition.id"))); String messageTraceparent = (String) message.getProperties().get("traceparent"); if (messageTraceparent == null) { From 8ab1158f1790c7d4a5f0b7906315a6a9e6692dea Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 20:45:30 -0700 Subject: [PATCH 11/21] remove global otel --- .../eventhubs/TracingIntegrationTests.java | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index bae947c0a0886..a2aeeb1bb4c2a 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -68,7 +68,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -@Isolated("Sets global TracingProvider.") @Execution(ExecutionMode.SAME_THREAD) public class TracingIntegrationTests extends IntegrationTestBase { private static final byte[] CONTENTS_BYTES = "Some-contents".getBytes(StandardCharsets.UTF_8); @@ -85,6 +84,7 @@ public class TracingIntegrationTests extends IntegrationTestBase { private EventProcessorClient processor; private Instant testStartTime; private EventData data; + private ClientOptions clientOptions; public TracingIntegrationTests() { super(new ClientLogger(TracingIntegrationTests.class)); @@ -92,8 +92,18 @@ public TracingIntegrationTests() { @Override protected void beforeTest() { + GlobalOpenTelemetry.resetForTest(); testStartTime = Instant.now().minusSeconds(1); data = new EventData(CONTENTS_BYTES); + spanProcessor = new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), testName); + OpenTelemetrySdk otel = OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .addSpanProcessor(spanProcessor) + .build()) + .build(); + + clientOptions = new ClientOptions().setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); } private Configuration createConfiguration(String v2Stack) { @@ -109,37 +119,22 @@ private Configuration createConfiguration(String v2Stack) { .build(); } - private void createClients(OpenTelemetrySdk otel, String v2Stack) { - GlobalOpenTelemetry.resetForTest(); - dispose(); - - spanProcessor = toClose(new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), testName)); - - ClientOptions options = new ClientOptions(); - if (otel == null) { - otel = OpenTelemetrySdk.builder() - .setTracerProvider( - SdkTracerProvider.builder() - .addSpanProcessor(spanProcessor) - .build()) - .buildAndRegisterGlobal(); - } - options.setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); + private void createClients(String v2Stack) { Configuration config = createConfiguration(v2Stack); producer = toClose(createBuilder() - .clientOptions(options) + .clientOptions(clientOptions) .configuration(config) .buildAsyncProducerClient()); consumer = toClose(createBuilder() - .clientOptions(options) + .clientOptions(clientOptions) .configuration(config) .consumerGroup("$Default") .buildAsyncConsumerClient()); consumerSync = toClose(createBuilder() - .clientOptions(options) + .clientOptions(clientOptions) .configuration(config) .consumerGroup("$Default") .buildConsumerClient()); @@ -147,14 +142,14 @@ private void createClients(OpenTelemetrySdk otel, String v2Stack) { @Override protected void afterTest() { - GlobalOpenTelemetry.resetForTest(); + spanProcessor.close(); } @ParameterizedTest @ValueSource(strings = {"true", "false"}) @NullSource public void sendAndReceiveFromPartition(String v2) throws InterruptedException { - createClients(null, v2); + createClients(v2); AtomicReference receivedMessage = new AtomicReference<>(); AtomicReference receivedSpan = new AtomicReference<>(); @@ -191,7 +186,7 @@ public void sendAndReceiveFromPartition(String v2) throws InterruptedException { @Test public void sendAndReceive() throws InterruptedException { - createClients(null, null); + createClients(null); AtomicReference receivedMessage = new AtomicReference<>(); AtomicReference receivedSpan = new AtomicReference<>(); @@ -237,7 +232,8 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { .addSpanProcessor(customSpanProcessor) .build()) .build(); - createClients(otel, null); + clientOptions = new ClientOptions().setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); + createClients(null); CountDownLatch latch = new CountDownLatch(2); customSpanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); @@ -269,7 +265,7 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { @Test public void sendAndReceiveParallel() throws InterruptedException { - createClients(null, null); + createClients(null); int messageCount = 5; CountDownLatch latch = new CountDownLatch(messageCount); @@ -307,7 +303,7 @@ public void sendAndReceiveParallel() throws InterruptedException { @Test public void sendBuffered() throws InterruptedException { - createClients(null, null); + createClients(null); CountDownLatch latch = new CountDownLatch(3); spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS) || hasOperationName(span, SEND)); @@ -376,7 +372,7 @@ public void sendBuffered() throws InterruptedException { @ParameterizedTest @ValueSource(strings = {"true", "false"}) public void syncReceive(String v2) { - createClients(null, v2); + createClients(v2); StepVerifier.create(producer.createBatch(new CreateBatchOptions().setPartitionId(PARTITION_ID)) .map(b -> { b.tryAdd(new EventData(CONTENTS_BYTES)); @@ -400,7 +396,7 @@ public void syncReceive(String v2) { @Test public void syncReceiveWithOptions() { - createClients(null, null); + createClients(null); StepVerifier.create(producer.createBatch(new CreateBatchOptions().setPartitionId(PARTITION_ID)) .map(b -> { b.tryAdd(new EventData(CONTENTS_BYTES)); @@ -425,7 +421,7 @@ public void syncReceiveWithOptions() { @Test public void syncReceiveTimeout() { - createClients(null, null); + createClients(null); List receivedMessages = consumerSync.receiveFromPartition(PARTITION_ID, 2, EventPosition.fromEnqueuedTime(testStartTime), Duration.ofSeconds(1)) .stream().collect(toList()); @@ -440,7 +436,7 @@ public void syncReceiveTimeout() { @ParameterizedTest @ValueSource(strings = {"true", "false"}) public void sendAndProcess(String v2) throws InterruptedException { - createClients(null, v2); + createClients(v2); AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference receivedMessage = new AtomicReference<>(); @@ -455,6 +451,7 @@ public void sendAndProcess(String v2) throws InterruptedException { EventHubClientBuilder builder = createBuilder(); processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) + .clientOptions(clientOptions) .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") .configuration(createConfiguration(v2)) @@ -499,7 +496,7 @@ public void sendAndProcess(String v2) throws InterruptedException { @Test @SuppressWarnings("try") public void sendNotInstrumentedAndProcess() throws InterruptedException { - createClients(null, null); + createClients(null); EventHubProducerAsyncClient notInstrumentedProducer = toClose(createBuilder() .clientOptions(new ClientOptions().setTracingOptions(new TracingOptions().setEnabled(false))) .buildAsyncProducerClient()); @@ -525,6 +522,7 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { try (Scope scope = test.makeCurrent()) { processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) + .clientOptions(clientOptions) .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") .checkpointStore(new SampleCheckpointStore()) @@ -568,7 +566,7 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { @ParameterizedTest @ValueSource(strings = {"true", "false"}) public void sendAndProcessBatch(String v2) throws InterruptedException { - createClients(null, v2); + createClients(v2); EventData message1 = new EventData(CONTENTS_BYTES); EventData message2 = new EventData(CONTENTS_BYTES); @@ -585,6 +583,7 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) + .clientOptions(clientOptions) .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") .configuration(createConfiguration(v2)) @@ -633,7 +632,7 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { @Test public void sendProcessAndFail() throws InterruptedException { - createClients(null, null); + createClients(null); AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference> received = new AtomicReference<>(); @@ -649,6 +648,7 @@ public void sendProcessAndFail() throws InterruptedException { processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) + .clientOptions(clientOptions) .consumerGroup("$Default") .checkpointStore(new SampleCheckpointStore()) .processEventBatch(eb -> { From 2a29a9be835e300fcdab66f2f80f74c623391ab0 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 21:13:02 -0700 Subject: [PATCH 12/21] up --- .../com/azure/messaging/eventhubs/TracingIntegrationTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index a2aeeb1bb4c2a..0db7e859eaedc 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -120,6 +120,7 @@ private Configuration createConfiguration(String v2Stack) { } private void createClients(String v2Stack) { + dispose(); Configuration config = createConfiguration(v2Stack); producer = toClose(createBuilder() @@ -309,6 +310,7 @@ public void sendBuffered() throws InterruptedException { spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS) || hasOperationName(span, SEND)); EventHubBufferedProducerAsyncClient bufferedProducer = toClose(new EventHubBufferedProducerClientBuilder() + .clientOptions(clientOptions) .credential(TestUtils.getPipelineCredential(cachedCredential)) .eventHubName(getEventHubName()) .fullyQualifiedNamespace(getFullyQualifiedDomainName()) From 91e680f7df05535d1356bc23ec828fd23f61ba06 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 12 Sep 2024 22:38:30 -0700 Subject: [PATCH 13/21] up --- ...EventProcessorClientErrorHandlingTest.java | 2 +- .../eventhubs/TracingIntegrationTests.java | 155 +++++++++--------- 2 files changed, 79 insertions(+), 78 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientErrorHandlingTest.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientErrorHandlingTest.java index 7e755bf0c6e14..918152090d5b5 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientErrorHandlingTest.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/EventProcessorClientErrorHandlingTest.java @@ -105,7 +105,7 @@ public void testCheckpointStoreErrors(CheckpointStore checkpointStore) throws In Assertions.assertTrue(errorContext.getThrowable() instanceof IllegalStateException); }, tracer, null, processorOptions); client.start(); - boolean completed = countDownLatch.await(3, TimeUnit.SECONDS); + boolean completed = countDownLatch.await(10, TimeUnit.SECONDS); try { client.stop(); } catch (IllegalStateException ex) { diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index 0db7e859eaedc..bd5f3b79f0128 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -32,7 +32,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; -import org.junit.jupiter.api.parallel.Isolated; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.NullSource; import org.junit.jupiter.params.provider.ValueSource; @@ -78,13 +77,9 @@ public class TracingIntegrationTests extends IntegrationTestBase { private static final AttributeKey OPERATION_NAME_ATTRIBUTE = AttributeKey.stringKey("messaging.operation.name"); private static final AttributeKey OPERATION_TYPE_ATTRIBUTE = AttributeKey.stringKey("messaging.operation.type"); private TestSpanProcessor spanProcessor; - private EventHubProducerAsyncClient producer; - private EventHubConsumerAsyncClient consumer; - private EventHubConsumerClient consumerSync; - private EventProcessorClient processor; private Instant testStartTime; private EventData data; - private ClientOptions clientOptions; + private OpenTelemetrySdk otel; public TracingIntegrationTests() { super(new ClientLogger(TracingIntegrationTests.class)); @@ -95,62 +90,21 @@ protected void beforeTest() { GlobalOpenTelemetry.resetForTest(); testStartTime = Instant.now().minusSeconds(1); data = new EventData(CONTENTS_BYTES); - spanProcessor = new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), testName); - OpenTelemetrySdk otel = OpenTelemetrySdk.builder() + spanProcessor = toClose(new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), testName)); + otel = toClose(OpenTelemetrySdk.builder() .setTracerProvider( SdkTracerProvider.builder() .addSpanProcessor(spanProcessor) .build()) - .build(); - - clientOptions = new ClientOptions().setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); - } - - private Configuration createConfiguration(String v2Stack) { - ConfigurationBuilder configBuilder = new ConfigurationBuilder(); - - if (v2Stack == null) { - return configBuilder.build(); - } - - return configBuilder - .putProperty("com.azure.messaging.eventhubs.v2", v2Stack) - .putProperty("com.azure.core.amqp.cache", v2Stack) - .build(); - } - - private void createClients(String v2Stack) { - dispose(); - Configuration config = createConfiguration(v2Stack); - - producer = toClose(createBuilder() - .clientOptions(clientOptions) - .configuration(config) - .buildAsyncProducerClient()); - - consumer = toClose(createBuilder() - .clientOptions(clientOptions) - .configuration(config) - .consumerGroup("$Default") - .buildAsyncConsumerClient()); - - consumerSync = toClose(createBuilder() - .clientOptions(clientOptions) - .configuration(config) - .consumerGroup("$Default") - .buildConsumerClient()); - } - - @Override - protected void afterTest() { - spanProcessor.close(); + .build()); } @ParameterizedTest @ValueSource(strings = {"true", "false"}) @NullSource public void sendAndReceiveFromPartition(String v2) throws InterruptedException { - createClients(v2); + EventHubProducerAsyncClient producer = createProducer(v2); + EventHubConsumerAsyncClient consumer = createConsumer(v2); AtomicReference receivedMessage = new AtomicReference<>(); AtomicReference receivedSpan = new AtomicReference<>(); @@ -187,7 +141,8 @@ public void sendAndReceiveFromPartition(String v2) throws InterruptedException { @Test public void sendAndReceive() throws InterruptedException { - createClients(null); + EventHubProducerAsyncClient producer = createProducer(null); + EventHubConsumerAsyncClient consumer = createConsumer(null); AtomicReference receivedMessage = new AtomicReference<>(); AtomicReference receivedSpan = new AtomicReference<>(); @@ -228,13 +183,14 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { AtomicReference receivedSpan = new AtomicReference<>(); TestSpanProcessor customSpanProcessor = toClose(new TestSpanProcessor(getFullyQualifiedDomainName(), getEventHubName(), "sendAndReceiveCustomProvider")); - OpenTelemetrySdk otel = OpenTelemetrySdk.builder() + otel = toClose(OpenTelemetrySdk.builder() .setTracerProvider(SdkTracerProvider.builder() .addSpanProcessor(customSpanProcessor) .build()) - .build(); - clientOptions = new ClientOptions().setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); - createClients(null); + .build()); + EventHubProducerAsyncClient producer = createProducer(null); + EventHubConsumerAsyncClient consumer = createConsumer(null); + CountDownLatch latch = new CountDownLatch(2); customSpanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); @@ -266,7 +222,8 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { @Test public void sendAndReceiveParallel() throws InterruptedException { - createClients(null); + EventHubProducerAsyncClient producer = createProducer(null); + EventHubConsumerAsyncClient consumer = createConsumer(null); int messageCount = 5; CountDownLatch latch = new CountDownLatch(messageCount); @@ -304,13 +261,13 @@ public void sendAndReceiveParallel() throws InterruptedException { @Test public void sendBuffered() throws InterruptedException { - createClients(null); + EventHubConsumerAsyncClient consumer = createConsumer(null); CountDownLatch latch = new CountDownLatch(3); spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS) || hasOperationName(span, SEND)); EventHubBufferedProducerAsyncClient bufferedProducer = toClose(new EventHubBufferedProducerClientBuilder() - .clientOptions(clientOptions) + .clientOptions(getClientOptions()) .credential(TestUtils.getPipelineCredential(cachedCredential)) .eventHubName(getEventHubName()) .fullyQualifiedNamespace(getFullyQualifiedDomainName()) @@ -374,7 +331,9 @@ public void sendBuffered() throws InterruptedException { @ParameterizedTest @ValueSource(strings = {"true", "false"}) public void syncReceive(String v2) { - createClients(v2); + EventHubProducerAsyncClient producer = createProducer(v2); + EventHubConsumerClient consumerSync = createSyncConsumer(v2); + StepVerifier.create(producer.createBatch(new CreateBatchOptions().setPartitionId(PARTITION_ID)) .map(b -> { b.tryAdd(new EventData(CONTENTS_BYTES)); @@ -398,7 +357,9 @@ public void syncReceive(String v2) { @Test public void syncReceiveWithOptions() { - createClients(null); + EventHubProducerAsyncClient producer = createProducer(null); + EventHubConsumerClient consumerSync = createSyncConsumer(null); + StepVerifier.create(producer.createBatch(new CreateBatchOptions().setPartitionId(PARTITION_ID)) .map(b -> { b.tryAdd(new EventData(CONTENTS_BYTES)); @@ -423,7 +384,8 @@ public void syncReceiveWithOptions() { @Test public void syncReceiveTimeout() { - createClients(null); + EventHubProducerAsyncClient producer = createProducer(null); + EventHubConsumerClient consumerSync = createSyncConsumer(null); List receivedMessages = consumerSync.receiveFromPartition(PARTITION_ID, 2, EventPosition.fromEnqueuedTime(testStartTime), Duration.ofSeconds(1)) .stream().collect(toList()); @@ -438,7 +400,7 @@ public void syncReceiveTimeout() { @ParameterizedTest @ValueSource(strings = {"true", "false"}) public void sendAndProcess(String v2) throws InterruptedException { - createClients(v2); + EventHubProducerAsyncClient producer = createProducer(v2); AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference receivedMessage = new AtomicReference<>(); @@ -451,12 +413,12 @@ public void sendAndProcess(String v2) throws InterruptedException { .verify(DEFAULT_TIMEOUT); EventHubClientBuilder builder = createBuilder(); - processor = new EventProcessorClientBuilder() + EventProcessorClient processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) - .clientOptions(clientOptions) + .clientOptions(getClientOptions()) + .configuration(getConfiguration(v2)) .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") - .configuration(createConfiguration(v2)) .checkpointStore(new SampleCheckpointStore()) .processEvent(ec -> { if (currentInProcess.compareAndSet(null, Span.current())) { @@ -498,7 +460,6 @@ public void sendAndProcess(String v2) throws InterruptedException { @Test @SuppressWarnings("try") public void sendNotInstrumentedAndProcess() throws InterruptedException { - createClients(null); EventHubProducerAsyncClient notInstrumentedProducer = toClose(createBuilder() .clientOptions(new ClientOptions().setTracingOptions(new TracingOptions().setEnabled(false))) .buildAsyncProducerClient()); @@ -522,9 +483,9 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { EventHubClientBuilder builder = createBuilder(); try (Scope scope = test.makeCurrent()) { - processor = new EventProcessorClientBuilder() + EventProcessorClient processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) - .clientOptions(clientOptions) + .clientOptions(getClientOptions()) .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") .checkpointStore(new SampleCheckpointStore()) @@ -568,7 +529,7 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { @ParameterizedTest @ValueSource(strings = {"true", "false"}) public void sendAndProcessBatch(String v2) throws InterruptedException { - createClients(v2); + EventHubProducerAsyncClient producer = createProducer(v2); EventData message1 = new EventData(CONTENTS_BYTES); EventData message2 = new EventData(CONTENTS_BYTES); @@ -583,12 +544,12 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { EventHubClientBuilder builder = createBuilder(); - processor = new EventProcessorClientBuilder() + EventProcessorClient processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) - .clientOptions(clientOptions) + .clientOptions(getClientOptions()) + .configuration(getConfiguration(v2)) .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) .consumerGroup("$Default") - .configuration(createConfiguration(v2)) .checkpointStore(new SampleCheckpointStore()) .processEventBatch(eb -> { if (currentInProcess.compareAndSet(null, Span.current())) { @@ -634,7 +595,7 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { @Test public void sendProcessAndFail() throws InterruptedException { - createClients(null); + EventHubProducerAsyncClient producer = createProducer(null); AtomicReference currentInProcess = new AtomicReference<>(); AtomicReference> received = new AtomicReference<>(); @@ -647,10 +608,10 @@ public void sendProcessAndFail() throws InterruptedException { EventHubClientBuilder builder = createBuilder(); - processor = new EventProcessorClientBuilder() + EventProcessorClient processor = new EventProcessorClientBuilder() .credential(builder.getFullyQualifiedNamespace(), builder.getEventHubName(), builder.getCredentials()) .initialPartitionEventPosition(p -> EventPosition.fromEnqueuedTime(testStartTime)) - .clientOptions(clientOptions) + .clientOptions(getClientOptions()) .consumerGroup("$Default") .checkpointStore(new SampleCheckpointStore()) .processEventBatch(eb -> { @@ -807,4 +768,44 @@ private boolean hasOperationName(ReadableSpan span, OperationName operationName) return operationName.toString().equals(span.getAttribute(OPERATION_NAME_ATTRIBUTE)); } + + private Configuration getConfiguration(String v2Stack) { + ConfigurationBuilder configBuilder = new ConfigurationBuilder(); + + if (v2Stack == null) { + return configBuilder.build(); + } + + return configBuilder + .putProperty("com.azure.messaging.eventhubs.v2", v2Stack) + .putProperty("com.azure.core.amqp.cache", v2Stack) + .build(); + } + + private ClientOptions getClientOptions() { + return new ClientOptions().setTracingOptions(new OpenTelemetryTracingOptions().setOpenTelemetry(otel)); + } + + private EventHubProducerAsyncClient createProducer(String v2) { + return toClose(createBuilder() + .clientOptions(getClientOptions()) + .configuration(getConfiguration(v2)) + .buildAsyncProducerClient()); + } + + private EventHubConsumerAsyncClient createConsumer(String v2) { + return toClose(createBuilder() + .clientOptions(getClientOptions()) + .configuration(getConfiguration(v2)) + .consumerGroup("$Default") + .buildAsyncConsumerClient()); + } + + private EventHubConsumerClient createSyncConsumer(String v2) { + return toClose(createBuilder() + .clientOptions(getClientOptions()) + .configuration(getConfiguration(v2)) + .consumerGroup("$Default") + .buildConsumerClient()); + } } From a935086fc2d4ab736132c56af29e430f9ab0069e Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 13 Sep 2024 10:33:21 -0700 Subject: [PATCH 14/21] up --- .../com/azure/messaging/eventhubs/IntegrationTestBase.java | 4 +++- .../azure/messaging/eventhubs/TracingIntegrationTests.java | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java index 9a74270a3328e..7b89877cfdef9 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/IntegrationTestBase.java @@ -54,7 +54,8 @@ public abstract class IntegrationTestBase extends TestBase { // This is a good idea to do in any production application as well - no point in waiting too long protected static final AmqpRetryOptions RETRY_OPTIONS = new AmqpRetryOptions() .setTryTimeout(Duration.ofSeconds(3)) - .setMaxDelay(Duration.ofSeconds(1)) + .setDelay(Duration.ofSeconds(1)) + .setMaxDelay(Duration.ofSeconds(5)) .setMaxRetries(10); protected final ClientLogger logger; @@ -247,4 +248,5 @@ protected void dispose() { private void skipIfNotRecordMode() { Assumptions.assumeTrue(getTestMode() != TestMode.PLAYBACK, "Is not in RECORD/LIVE mode."); } + } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index bd5f3b79f0128..5dcdda41f16fb 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -32,6 +32,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.api.parallel.Isolated; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.NullSource; import org.junit.jupiter.params.provider.ValueSource; @@ -67,6 +68,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +@Isolated @Execution(ExecutionMode.SAME_THREAD) public class TracingIntegrationTests extends IntegrationTestBase { private static final byte[] CONTENTS_BYTES = "Some-contents".getBytes(StandardCharsets.UTF_8); From ca1af7136b8d3663b8f98070dc43095a47a406ce Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 13 Sep 2024 11:40:17 -0700 Subject: [PATCH 15/21] up --- .../eventhubs/EventProcessorClient.java | 9 ++++ .../eventhubs/TracingIntegrationTests.java | 43 +++++++++++-------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java index 2a842c89231ab..672c115cdc90d 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java @@ -339,6 +339,15 @@ public synchronized void stop(Duration timeout) { .block(timeout); } + synchronized boolean stop(long timeout, TimeUnit timeUnit) { + stop(); + try { + return scheduler.get().awaitTermination(timeout, timeUnit); + } catch (InterruptedException e) { + return false; + } + } + /** * Returns {@code true} if the event processor is running. If the event processor is already running, calling {@link * #start()} has no effect. diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index 5dcdda41f16fb..3b52d6eaf0c50 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -50,6 +50,7 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import static com.azure.messaging.eventhubs.TestUtils.getEventHubName; @@ -74,6 +75,7 @@ public class TracingIntegrationTests extends IntegrationTestBase { private static final byte[] CONTENTS_BYTES = "Some-contents".getBytes(StandardCharsets.UTF_8); private static final String PARTITION_ID = "0"; private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); + private static final AtomicLong ownerLevel = new AtomicLong(); private final AtomicReference cachedCredential = new AtomicReference<>(); private static final AttributeKey OPERATION_NAME_ATTRIBUTE = AttributeKey.stringKey("messaging.operation.name"); @@ -114,7 +116,7 @@ public void sendAndReceiveFromPartition(String v2) throws InterruptedException { CountDownLatch latch = new CountDownLatch(2); spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer - .receiveFromPartition(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime)) + .receiveFromPartition(PARTITION_ID, EventPosition.fromEnqueuedTime(testStartTime), getReceiveOptions()) .take(1) .subscribe(pe -> { if (receivedMessage.compareAndSet(null, pe)) { @@ -232,7 +234,7 @@ public void sendAndReceiveParallel() throws InterruptedException { spanProcessor.notifyIfCondition(latch, span -> hasOperationName(span, PROCESS)); - StepVerifier.create(producer.send(data, new SendOptions())) + StepVerifier.create(producer.send(data)) .expectComplete() .verify(DEFAULT_TIMEOUT); @@ -281,14 +283,13 @@ public void sendBuffered() throws InterruptedException { }) .buildAsyncClient()); - Instant start = Instant.now(); EventData event1 = new EventData("1"); EventData event2 = new EventData("2"); // Using a specific partition in the case that an epoch receiver was created // (i.e. EventHubConsumerAsyncClientIntegrationTest), which this scenario will fail when trying to create a // receiver. - SendOptions sendOptions = new SendOptions().setPartitionId("0"); + SendOptions sendOptions = new SendOptions().setPartitionId(PARTITION_ID); Boolean partitionIdExists = bufferedProducer.getPartitionIds() .any(id -> id.equals(sendOptions.getPartitionId())) .block(Duration.ofSeconds(30)); @@ -302,7 +303,7 @@ public void sendBuffered() throws InterruptedException { .verify(DEFAULT_TIMEOUT); StepVerifier.create(consumer - .receiveFromPartition(sendOptions.getPartitionId(), EventPosition.fromEnqueuedTime(start)) + .receiveFromPartition(sendOptions.getPartitionId(), EventPosition.fromEnqueuedTime(testStartTime), getReceiveOptions()) .map(e -> { logger.atInfo() .addKeyValue("event", e.getData().getBodyAsString()) @@ -346,7 +347,11 @@ public void syncReceive(String v2) { .expectComplete() .verify(DEFAULT_TIMEOUT); - List receivedMessages = consumerSync.receiveFromPartition(PARTITION_ID, 2, EventPosition.fromEnqueuedTime(testStartTime), Duration.ofSeconds(10)) + List receivedMessages = consumerSync.receiveFromPartition(PARTITION_ID, + 2, + EventPosition.fromEnqueuedTime(testStartTime), + Duration.ofSeconds(10), + getReceiveOptions()) .stream().collect(toList()); assertEquals(2, receivedMessages.size()); @@ -373,7 +378,7 @@ public void syncReceiveWithOptions() { .verify(DEFAULT_TIMEOUT); List receivedMessages = consumerSync.receiveFromPartition(PARTITION_ID, 2, - EventPosition.fromEnqueuedTime(testStartTime), Duration.ofSeconds(10), new ReceiveOptions()) + EventPosition.fromEnqueuedTime(testStartTime), Duration.ofSeconds(10), getReceiveOptions()) .stream().collect(toList()); assertEquals(2, receivedMessages.size()); @@ -386,10 +391,9 @@ public void syncReceiveWithOptions() { @Test public void syncReceiveTimeout() { - EventHubProducerAsyncClient producer = createProducer(null); EventHubConsumerClient consumerSync = createSyncConsumer(null); List receivedMessages = consumerSync.receiveFromPartition(PARTITION_ID, 2, - EventPosition.fromEnqueuedTime(testStartTime), Duration.ofSeconds(1)) + EventPosition.fromEnqueuedTime(testStartTime), Duration.ofSeconds(1), getReceiveOptions()) .stream().collect(toList()); List spans = spanProcessor.getEndedSpans(); @@ -431,10 +435,10 @@ public void sendAndProcess(String v2) throws InterruptedException { .processError(e -> fail("unexpected error", e.getThrowable())) .buildEventProcessorClient(); - toClose((Closeable) () -> processor.stop()); + toClose((Closeable) () -> processor.stop(10, TimeUnit.SECONDS)); processor.start(); assertTrue(latch.await(30, TimeUnit.SECONDS)); - processor.stop(); + assertTrue(processor.stop(10, TimeUnit.SECONDS)); assertTrue(currentInProcess.get().getSpanContext().isValid()); List spans = spanProcessor.getEndedSpans(); @@ -500,11 +504,11 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { .processError(e -> fail("unexpected error", e.getThrowable())) .buildEventProcessorClient(); - toClose((Closeable) () -> processor.stop()); + toClose((Closeable) () -> processor.stop(10, TimeUnit.SECONDS)); processor.start(); assertTrue(latch.await(30, TimeUnit.SECONDS)); - processor.stop(); + processor.stop(10, TimeUnit.SECONDS); } List spans = spanProcessor.getEndedSpans(); @@ -567,10 +571,10 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { }, 2) .processError(e -> fail("unexpected error", e.getThrowable())) .buildEventProcessorClient(); - toClose((Closeable) () -> processor.stop()); + toClose((Closeable) () -> processor.stop(10, TimeUnit.SECONDS)); processor.start(); assertTrue(latch.await(30, TimeUnit.SECONDS)); - processor.stop(); + processor.stop(10, TimeUnit.SECONDS); List spans = spanProcessor.getEndedSpans(); @@ -626,10 +630,10 @@ public void sendProcessAndFail() throws InterruptedException { .processError(e -> fail("unexpected error", e.getThrowable())) .buildEventProcessorClient(); - toClose((Closeable) () -> processor.stop()); + toClose((Closeable) () -> processor.stop(10, TimeUnit.SECONDS)); processor.start(); assertTrue(latch.await(30, TimeUnit.SECONDS)); - processor.stop(); + processor.stop(10, TimeUnit.SECONDS); List spans = spanProcessor.getEndedSpans(); List processed = findSpans(spans, PROCESS) @@ -810,4 +814,9 @@ private EventHubConsumerClient createSyncConsumer(String v2) { .consumerGroup("$Default") .buildConsumerClient()); } + + private ReceiveOptions getReceiveOptions() { + return new ReceiveOptions() + .setOwnerLevel(ownerLevel.getAndIncrement()); + } } From 9047e917aded5f6e9b1dcb45750f0065530eb21f Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 13 Sep 2024 12:16:51 -0700 Subject: [PATCH 16/21] up --- .../azure/messaging/eventhubs/TracingIntegrationTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index 3b52d6eaf0c50..f5f9d6007c646 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -154,7 +154,7 @@ public void sendAndReceive() throws InterruptedException { CountDownLatch latch = new CountDownLatch(2); spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer - .receive() + .receive(false, getReceiveOptions()) .take(DEFAULT_TIMEOUT) .subscribe(pe -> { if (receivedMessage.compareAndSet(null, pe)) { @@ -198,7 +198,7 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { CountDownLatch latch = new CountDownLatch(2); customSpanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); - toClose(consumer.receive() + toClose(consumer.receive(false, getReceiveOptions()) .take(1) .subscribe(pe -> { if (receivedMessage.compareAndSet(null, pe)) { @@ -239,7 +239,7 @@ public void sendAndReceiveParallel() throws InterruptedException { .verify(DEFAULT_TIMEOUT); StepVerifier.create(consumer - .receive() + .receive(false, getReceiveOptions()) .take(messageCount) .doOnNext(pe -> { String traceparent = (String) pe.getData().getProperties().get("traceparent"); From e0de76fd017e6362b1a7e1a033b4aaf35e570e64 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 13 Sep 2024 13:04:56 -0700 Subject: [PATCH 17/21] up --- .../azure/messaging/eventhubs/TracingIntegrationTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index f5f9d6007c646..e279d9a6c7307 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -154,7 +154,7 @@ public void sendAndReceive() throws InterruptedException { CountDownLatch latch = new CountDownLatch(2); spanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); toClose(consumer - .receive(false, getReceiveOptions()) + .receive(true, getReceiveOptions()) .take(DEFAULT_TIMEOUT) .subscribe(pe -> { if (receivedMessage.compareAndSet(null, pe)) { @@ -198,7 +198,7 @@ public void sendAndReceiveCustomProvider() throws InterruptedException { CountDownLatch latch = new CountDownLatch(2); customSpanProcessor.notifyIfCondition(latch, span -> span == receivedSpan.get() || hasOperationName(span, SEND)); - toClose(consumer.receive(false, getReceiveOptions()) + toClose(consumer.receive(true, getReceiveOptions()) .take(1) .subscribe(pe -> { if (receivedMessage.compareAndSet(null, pe)) { @@ -239,7 +239,7 @@ public void sendAndReceiveParallel() throws InterruptedException { .verify(DEFAULT_TIMEOUT); StepVerifier.create(consumer - .receive(false, getReceiveOptions()) + .receive(true, getReceiveOptions()) .take(messageCount) .doOnNext(pe -> { String traceparent = (String) pe.getData().getProperties().get("traceparent"); From 12de3a8ddd3d406a716664fad18a1df6c10a6070 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Mon, 16 Sep 2024 15:35:29 -0700 Subject: [PATCH 18/21] lint and stress test fixes --- sdk/eventhubs/azure-messaging-eventhubs-stress/Dockerfile | 4 ++++ .../com/azure/messaging/eventhubs/MessageFluxWrapper.java | 1 - .../azure/messaging/eventhubs/TracingIntegrationTests.java | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs-stress/Dockerfile b/sdk/eventhubs/azure-messaging-eventhubs-stress/Dockerfile index 51ac85c415198..119d735cbda57 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs-stress/Dockerfile +++ b/sdk/eventhubs/azure-messaging-eventhubs-stress/Dockerfile @@ -19,6 +19,10 @@ RUN --mount=type=cache,target=/root/.m2 \ mvn -f /stress-eh/eng/code-quality-reports/pom.xml clean install ${SKIP_CHECKS} && \ mvn -f /stress-eh/sdk/tools/pom.xml clean install ${SKIP_CHECKS} && \ mvn -f /stress-eh/sdk/core/azure-core/pom.xml clean install ${SKIP_CHECKS} && \ + mvn -f /stress-eh/sdk/core/azure-core-http-netty/pom.xml clean install ${SKIP_CHECKS} && \ + mvn -f /stress-eh/sdk/core/azure-core-http-okhttp/pom.xml clean install ${SKIP_CHECKS} && \ + mvn -f /stress-eh/sdk/core/azure-core-http-vertx/pom.xml clean install ${SKIP_CHECKS} && \ + mvn -f /stress-eh/sdk/core/azure-core-http-jdk-httpclient/pom.xml clean install ${SKIP_CHECKS} && \ mvn -f /stress-eh/sdk/core/azure-core-test/pom.xml clean install ${SKIP_CHECKS} && \ mvn -f /stress-eh/sdk/core/azure-core-amqp/pom.xml clean install ${SKIP_CHECKS} && \ mvn -f /stress-eh/sdk/core/azure-core-http-netty/pom.xml clean install ${SKIP_CHECKS} && \ diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java index 450e68212eefc..cc26669c98794 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/MessageFluxWrapper.java @@ -3,7 +3,6 @@ package com.azure.messaging.eventhubs; -import com.azure.core.amqp.implementation.MessageFlux; import com.azure.messaging.eventhubs.implementation.AmqpReceiveLinkProcessor; import org.apache.qpid.proton.message.Message; import reactor.core.publisher.Flux; diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index e279d9a6c7307..a1784266ef959 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -75,7 +75,7 @@ public class TracingIntegrationTests extends IntegrationTestBase { private static final byte[] CONTENTS_BYTES = "Some-contents".getBytes(StandardCharsets.UTF_8); private static final String PARTITION_ID = "0"; private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); - private static final AtomicLong ownerLevel = new AtomicLong(); + private static final AtomicLong OWNER_LEVEL = new AtomicLong(); private final AtomicReference cachedCredential = new AtomicReference<>(); private static final AttributeKey OPERATION_NAME_ATTRIBUTE = AttributeKey.stringKey("messaging.operation.name"); @@ -817,6 +817,6 @@ private EventHubConsumerClient createSyncConsumer(String v2) { private ReceiveOptions getReceiveOptions() { return new ReceiveOptions() - .setOwnerLevel(ownerLevel.getAndIncrement()); + .setOwnerLevel(OWNER_LEVEL.getAndIncrement()); } } From 4c38952c57ec96d129fdb260f77dd11d1697de5f Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Wed, 18 Sep 2024 13:24:50 -0700 Subject: [PATCH 19/21] Rebase and review comments --- sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md | 5 +++-- .../messaging/eventhubs/EventHubConsumerClient.java | 2 +- .../eventhubs/EventHubsProducerInstrumentation.java | 10 ++-------- .../EventHubsConsumerInstrumentation.java | 2 +- .../instrumentation/EventHubsMetricsProvider.java | 4 ++-- .../instrumentation/EventHubsTracer.java | 2 +- .../instrumentation/InstrumentationScope.java | 2 +- .../instrumentation/InstrumentationUtils.java | 3 +++ 8 files changed, 14 insertions(+), 16 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md b/sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md index 6161f64c1851f..acdc0f41c37e5 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md +++ b/sdk/eventhubs/azure-messaging-eventhubs/CHANGELOG.md @@ -7,7 +7,7 @@ - Integrated RequestResponseChannelCache (CBS, Management channel cache) and ReactorSessionCache, these caches are activated when V2 stack is opted-in using the configuration `com.azure.messaging.eventhubs.v2`. ([39107](https://github.com/Azure/azure-sdk-for-java/pull/39107)), ([41805](https://github.com/Azure/azure-sdk-for-java/pull/41805)) - Add `EventProcessorClient.stop(Duration timeout)` to stop the `EventProcessorClient` and await it to shut down. The `EventProcessorClient.stop()` method now will wait for up to the default timeout (10 seconds) waiting for the processor to stop. ([#41878](https://github.com/Azure/azure-sdk-for-java/pull/41878)) -- Observability improvements +- Observability improvements ([#38899](https://github.com/Azure/azure-sdk-for-java/pull/38899)) - Added span for update checkpoint call. - Added partitionId and consumer group (when available) to span and metric attributes. @@ -15,7 +15,8 @@ - Updated distributed traces and metrics to follow OpenTelemetry semantic conventions 1.27.0. Please refer to [OpenTelemetry messaging semantic conventions](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/messaging) for more details. - - Span names now follow `{entity name} {operaiton name}` pattern + ([#38899](https://github.com/Azure/azure-sdk-for-java/pull/38899)) + - Span names now follow `{operaiton name} {entity name}` pattern - Updated metric names and attributes ### Bugs Fixed diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java index c77b944761e5d..b040109a02be1 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java @@ -260,7 +260,7 @@ public IterableStream receiveFromPartition(String partitionId, i } if (consumer.isV2()) { - // Sync receiver instrumentation is implemented in the SynchronousReceiver class + // Sync receiver instrumentation is implemented in the SynchronousPartitionReceiver class return syncReceiver.receive(partitionId, startingPosition, defaultReceiveOptions, maximumMessageCount, maximumWaitTime); } diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java index 6fa76e15dc8b3..2d2a0a0eabe6e 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubsProducerInstrumentation.java @@ -13,9 +13,6 @@ import com.azure.messaging.eventhubs.implementation.instrumentation.OperationName; import reactor.core.publisher.Mono; - -import java.util.function.BiConsumer; - import static com.azure.core.util.tracing.SpanKind.CLIENT; import static com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY; import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_BATCH_MESSAGE_COUNT; @@ -35,12 +32,9 @@ Mono sendBatch(Mono publisher, EventDataBatch batch) { return publisher; } - BiConsumer reportMetricsCallback = (m, s) -> - m.reportBatchSend(batch.getCount(), batch.getPartitionId(), s); - - return Mono.using( - () -> new InstrumentationScope(tracer, meter, reportMetricsCallback) + () -> new InstrumentationScope(tracer, meter, + (m, s) -> m.reportBatchSend(batch.getCount(), batch.getPartitionId(), s)) .setSpan(startPublishSpanWithLinks(batch)), scope -> publisher .doOnError(scope::setError) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java index 08fdda39a71eb..2d2219d9186dc 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsConsumerInstrumentation.java @@ -28,7 +28,7 @@ import static com.azure.messaging.eventhubs.implementation.instrumentation.InstrumentationUtils.MESSAGING_DESTINATION_PARTITION_ID; import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.RECEIVE; -public class EventHubsConsumerInstrumentation { +public final class EventHubsConsumerInstrumentation { private static final Symbol ENQUEUED_TIME_UTC_ANNOTATION_NAME_SYMBOL = Symbol.valueOf(ENQUEUED_TIME_UTC_ANNOTATION_NAME.getValue()); private static final InstrumentationScope NOOP_SCOPE = new InstrumentationScope(null, null, null); private final EventHubsTracer tracer; diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java index fa1a46d0c987d..c74a825ebfe45 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsMetricsProvider.java @@ -37,7 +37,7 @@ import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.RECEIVE; import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.SEND; -public class EventHubsMetricsProvider { +public final class EventHubsMetricsProvider { private final Meter meter; private final boolean isEnabled; private static final ClientLogger LOGGER = new ClientLogger(EventHubsMetricsProvider.class); @@ -168,7 +168,7 @@ private TelemetryAttributes getOrCreateAttributes(OperationName operationName, S } private Map getCommonAttributes(String namespace, String entityName, String consumerGroup) { - Map commonAttributesMap = new HashMap<>(3); + Map commonAttributesMap = new HashMap<>(4); commonAttributesMap.put(MESSAGING_SYSTEM, MESSAGING_SYSTEM_VALUE); commonAttributesMap.put(SERVER_ADDRESS, namespace); commonAttributesMap.put(MESSAGING_DESTINATION_NAME, entityName); diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java index b1de58cba414f..e023152726434 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/EventHubsTracer.java @@ -45,7 +45,7 @@ import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.EVENT; import static com.azure.messaging.eventhubs.implementation.instrumentation.OperationName.PROCESS; -public class EventHubsTracer { +public final class EventHubsTracer { private static final AutoCloseable NOOP_AUTOCLOSEABLE = () -> { }; diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java index 3b3987f32193f..5e7e05141ab43 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationScope.java @@ -23,7 +23,7 @@ public final class InstrumentationScope implements AutoCloseable { private String errorType; private Context span = Context.NONE; private AutoCloseable spanScope; - private boolean closed = false; + private volatile boolean closed = false; public InstrumentationScope(EventHubsTracer tracer, EventHubsMetricsProvider meter, diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java index d3f6521f9fa0a..fcc4128b6bcfb 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/implementation/instrumentation/InstrumentationUtils.java @@ -94,6 +94,9 @@ public static String getOperationType(OperationName name) { case PROCESS: return "process"; default: + // other operations are not documented in the otel semantic conventions + // they are not 'core' messaging operations and therefor don't have type + // we'll still report operation name attribute for them. return null; } } From 1f3091df9e8a84ea87969ed0b3a2d548235c3d02 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Wed, 18 Sep 2024 13:29:54 -0700 Subject: [PATCH 20/21] Rebase and review comments --- .../eventhubs/EventProcessorClient.java | 9 --------- .../eventhubs/TracingIntegrationTests.java | 16 ++++++++-------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java index 672c115cdc90d..2a842c89231ab 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventProcessorClient.java @@ -339,15 +339,6 @@ public synchronized void stop(Duration timeout) { .block(timeout); } - synchronized boolean stop(long timeout, TimeUnit timeUnit) { - stop(); - try { - return scheduler.get().awaitTermination(timeout, timeUnit); - } catch (InterruptedException e) { - return false; - } - } - /** * Returns {@code true} if the event processor is running. If the event processor is already running, calling {@link * #start()} has no effect. diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java index a1784266ef959..12e181cdfd0a6 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/test/java/com/azure/messaging/eventhubs/TracingIntegrationTests.java @@ -435,10 +435,10 @@ public void sendAndProcess(String v2) throws InterruptedException { .processError(e -> fail("unexpected error", e.getThrowable())) .buildEventProcessorClient(); - toClose((Closeable) () -> processor.stop(10, TimeUnit.SECONDS)); + toClose((Closeable) () -> processor.stop()); processor.start(); assertTrue(latch.await(30, TimeUnit.SECONDS)); - assertTrue(processor.stop(10, TimeUnit.SECONDS)); + processor.stop(); assertTrue(currentInProcess.get().getSpanContext().isValid()); List spans = spanProcessor.getEndedSpans(); @@ -504,11 +504,11 @@ public void sendNotInstrumentedAndProcess() throws InterruptedException { .processError(e -> fail("unexpected error", e.getThrowable())) .buildEventProcessorClient(); - toClose((Closeable) () -> processor.stop(10, TimeUnit.SECONDS)); + toClose((Closeable) () -> processor.stop()); processor.start(); assertTrue(latch.await(30, TimeUnit.SECONDS)); - processor.stop(10, TimeUnit.SECONDS); + processor.stop(); } List spans = spanProcessor.getEndedSpans(); @@ -571,10 +571,10 @@ public void sendAndProcessBatch(String v2) throws InterruptedException { }, 2) .processError(e -> fail("unexpected error", e.getThrowable())) .buildEventProcessorClient(); - toClose((Closeable) () -> processor.stop(10, TimeUnit.SECONDS)); + toClose((Closeable) () -> processor.stop()); processor.start(); assertTrue(latch.await(30, TimeUnit.SECONDS)); - processor.stop(10, TimeUnit.SECONDS); + processor.stop(); List spans = spanProcessor.getEndedSpans(); @@ -630,10 +630,10 @@ public void sendProcessAndFail() throws InterruptedException { .processError(e -> fail("unexpected error", e.getThrowable())) .buildEventProcessorClient(); - toClose((Closeable) () -> processor.stop(10, TimeUnit.SECONDS)); + toClose((Closeable) () -> processor.stop()); processor.start(); assertTrue(latch.await(30, TimeUnit.SECONDS)); - processor.stop(10, TimeUnit.SECONDS); + processor.stop(); List spans = spanProcessor.getEndedSpans(); List processed = findSpans(spans, PROCESS) From 762d7c40af0089e4635cb78a5c9df6f58a445879 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Wed, 18 Sep 2024 15:18:43 -0700 Subject: [PATCH 21/21] nits --- .../azure-messaging-eventhubs-checkpointstore-blob/CHANGELOG.md | 1 + .../com/azure/messaging/eventhubs/EventHubConsumerClient.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/CHANGELOG.md b/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/CHANGELOG.md index fb15906019ff0..c88cc6ccc7e30 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/CHANGELOG.md +++ b/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/CHANGELOG.md @@ -7,6 +7,7 @@ ### Breaking Changes - Experimental checkpointing metrics are no longer reported by this package. They've been moved to `azure-messaging-eventhubs` package. + ([#38899](https://github.com/Azure/azure-sdk-for-java/pull/38899)) ### Bugs Fixed diff --git a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java index b040109a02be1..003eb4a16d620 100644 --- a/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java +++ b/sdk/eventhubs/azure-messaging-eventhubs/src/main/java/com/azure/messaging/eventhubs/EventHubConsumerClient.java @@ -315,7 +315,7 @@ public IterableStream receiveFromPartition(String partitionId, i } if (consumer.isV2()) { - // Sync receiver instrumentation is implemented in the SynchronousReceiver class + // Sync receiver instrumentation is implemented in the SynchronousPartitionReceiver class return syncReceiver.receive(partitionId, startingPosition, receiveOptions, maximumMessageCount, maximumWaitTime); }