From 2e07dde57043ea3c0877ceca853c526b8bf0e5ff Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Thu, 26 Jun 2025 19:14:41 -0700 Subject: [PATCH 01/39] Add AWS CloudWatch integration through Event Listener --- .../core/config/FeatureConfiguration.java | 21 ++ .../src/main/resources/application.properties | 17 +- .../quarkus/events/AwsCloudwatchConfig.java | 26 +++ ...rkusPolarisEventListenerConfiguration.java | 18 +- service/common/build.gradle.kts | 1 + .../service/admin/PolarisServiceImpl.java | 9 +- .../events/AfterCatalogCreatedEvent.java | 29 +++ .../events/AwsCloudWatchEventListener.java | 212 ++++++++++++++++++ .../events/EventListenerConfiguration.java | 30 +++ .../service/events/PolarisEventListener.java | 5 + .../apache/polaris/service/TestServices.java | 3 +- 11 files changed, 365 insertions(+), 6 deletions(-) create mode 100644 runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/events/AfterCatalogCreatedEvent.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index fe265c3072..caf1c5222f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -298,4 +298,25 @@ public static void enforceFeatureEnabledOrThrow( + "in their snapshot summary") .defaultValue(false) .buildFeatureConfiguration(); + + public static final FeatureConfiguration AWS_CLOUDWATCH_EVENT_LISTENER_LOG_GROUP = + PolarisConfiguration.builder() + .key("AWS_CLOUDWATCH_EVENT_LISTENER_LOG_GROUP") + .description("The log group you'd like the AWS CloudWatch Event Listener to output to.") + .defaultValue("polaris-log-group") + .buildFeatureConfiguration(); + + public static final FeatureConfiguration AWS_CLOUDWATCH_EVENT_LISTENER_LOG_STREAM = + PolarisConfiguration.builder() + .key("AWS_CLOUDWATCH_EVENT_LISTENER_LOG_STREAM") + .description("The log stream you'd like the AWS CloudWatch Event Listener to output to.") + .defaultValue("polaris-log-stream") + .buildFeatureConfiguration(); + + public static final FeatureConfiguration AWS_CLOUDWATCH_EVENT_LISTENER_REGION = + PolarisConfiguration.builder() + .key("AWS_CLOUDWATCH_EVENT_LISTENER_REGION") + .description("The region where the log group is located for the AWS CloudWatch Event Listener.") + .defaultValue("us-east-1") + .buildFeatureConfiguration(); } diff --git a/runtime/defaults/src/main/resources/application.properties b/runtime/defaults/src/main/resources/application.properties index 592501f76d..508f4e51cb 100644 --- a/runtime/defaults/src/main/resources/application.properties +++ b/runtime/defaults/src/main/resources/application.properties @@ -109,7 +109,9 @@ polaris.realm-context.header-name=Polaris-Realm polaris.realm-context.require-header=false polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false -polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"] +polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] +polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true +polaris.readiness.ignore-severe-issues=true # polaris.features."ENABLE_CATALOG_FEDERATION"=true polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] @@ -118,13 +120,22 @@ polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] # polaris.persistence.type=eclipse-link # polaris.persistence.type=in-memory-atomic -polaris.persistence.type=in-memory +# polaris.persistence.type=in-memory +polaris.persistence.type=relational-jdbc +quarkus.datasource.db-kind=pgsql +quarkus.datasource.jdbc.url=jdbc:postgresql://localhost:5432/POLARIS +quarkus.datasource.username=postgres +quarkus.datasource.password=postgres polaris.secrets-manager.type=in-memory polaris.file-io.type=default -polaris.event-listener.type=no-op +# polaris.event-listener.type=no-op +# polaris.event-listener.type=aws-cloudwatch +# polaris.event-listener.aws-cloudwatch.log-group=test-group +# polaris.event-listener.aws-cloudwatch.log-stream=test-stream +# polaris.event-listener.aws-cloudwatch.region=us-west-2 polaris.log.request-id-header-name=Polaris-Request-Id # polaris.log.mdc.aid=polaris diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java new file mode 100644 index 0000000000..e0da71140c --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.events; + +public interface AwsCloudwatchConfig { + String logGroup(); + String logStream(); + String region(); +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java index 8921c726c6..b79556ca13 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java @@ -20,13 +20,29 @@ import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; +import org.apache.polaris.service.events.EventListenerConfiguration; + +import java.util.Optional; @StaticInitSafe @ConfigMapping(prefix = "polaris.event-listener") -public interface QuarkusPolarisEventListenerConfiguration { +public interface QuarkusPolarisEventListenerConfiguration extends EventListenerConfiguration { /** * The type of the event listener to use. Must be a registered {@link * org.apache.polaris.service.events.PolarisEventListener} identifier. */ String type(); + + @WithName("aws-cloudwatch.log-group") + @Override + Optional awsCloudwatchlogGroup(); + + @WithName("aws-cloudwatch.log-stream") + @Override + Optional awsCloudwatchlogStream(); + + @WithName("aws-cloudwatch.region") + @Override + Optional awsCloudwatchRegion(); } diff --git a/service/common/build.gradle.kts b/service/common/build.gradle.kts index 2f5d958259..21f8886b8a 100644 --- a/service/common/build.gradle.kts +++ b/service/common/build.gradle.kts @@ -82,6 +82,7 @@ dependencies { implementation("software.amazon.awssdk:sts") implementation("software.amazon.awssdk:iam-policy-builder") implementation("software.amazon.awssdk:s3") + implementation("software.amazon.awssdk:cloudwatchlogs") implementation(platform(libs.azuresdk.bom)) implementation("com.azure:azure-core") diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 719e6d44b0..aded97444d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -78,6 +78,8 @@ import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.config.ReservedProperties; +import org.apache.polaris.service.events.AfterCatalogCreatedEvent; +import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.types.PolicyIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,6 +97,7 @@ public class PolarisServiceImpl private final UserSecretsManagerFactory userSecretsManagerFactory; private final CallContext callContext; private final ReservedProperties reservedProperties; + private final PolarisEventListener polarisEventListener; @Inject public PolarisServiceImpl( @@ -103,13 +106,15 @@ public PolarisServiceImpl( UserSecretsManagerFactory userSecretsManagerFactory, PolarisAuthorizer polarisAuthorizer, CallContext callContext, - ReservedProperties reservedProperties) { + ReservedProperties reservedProperties, + PolarisEventListener polarisEventListener) { this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; this.userSecretsManagerFactory = userSecretsManagerFactory; this.polarisAuthorizer = polarisAuthorizer; this.callContext = callContext; this.reservedProperties = reservedProperties; + this.polarisEventListener = polarisEventListener; // FIXME: This is a hack to set the current context for downstream calls. CallContext.setCurrentContext(callContext); } @@ -148,6 +153,8 @@ public Response createCatalog( validateConnectionConfigInfo(catalog); Catalog newCatalog = new CatalogEntity(adminService.createCatalog(request)).asCatalog(); LOGGER.info("Created new catalog {}", newCatalog); + polarisEventListener.onAfterCatalogCreated( + new AfterCatalogCreatedEvent(newCatalog), callContext); return Response.status(Response.Status.CREATED).build(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AfterCatalogCreatedEvent.java b/service/common/src/main/java/org/apache/polaris/service/events/AfterCatalogCreatedEvent.java new file mode 100644 index 0000000000..300ae1a4e2 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/events/AfterCatalogCreatedEvent.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.events; + +import org.apache.polaris.core.admin.model.Catalog; + +/** + * Emitted after Polaris creates a catalog (internal or external). This is not emitted if there's an + * exception while created. + * + * @param catalog The catalog that was created + */ +public record AfterCatalogCreatedEvent(Catalog catalog) implements PolarisEvent {} diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java new file mode 100644 index 0000000000..9af9bca970 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.events; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.smallrye.common.annotation.Identifier; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import org.apache.polaris.core.context.CallContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; +import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.InputLogEvent; +import software.amazon.awssdk.services.cloudwatchlogs.model.InvalidSequenceTokenException; +import software.amazon.awssdk.services.cloudwatchlogs.model.LogStream; +import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.ResourceAlreadyExistsException; + +@ApplicationScoped +@Identifier("aws-cloudwatch") +public class AwsCloudWatchEventListener extends PolarisEventListener { + private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListener.class); + private final ObjectMapper objectMapper = new ObjectMapper(); + private static final int MAX_BATCH_SIZE = 10_000; + private static final int MAX_WAIT_MS = 5000; + + private final BlockingQueue queue = new LinkedBlockingQueue<>(); + private CloudWatchLogsClient client; + private volatile String sequenceToken; + + private volatile boolean running = true; + + ExecutorService executorService; + + private Future backgroundTask; + + private final String logGroup; + private final String logStream; + private final Region region; + + @Inject + public AwsCloudWatchEventListener(EventListenerConfiguration config, ExecutorService executorService) { + this.executorService = executorService; + + this.logStream = config.awsCloudwatchlogStream().orElse("polaris-cloudwatch-default-stream"); + this.logGroup = config.awsCloudwatchlogGroup().orElse("polaris-cloudwatch-default-group"); + this.region = Region.of(config.awsCloudwatchRegion().orElse("us-east-1")); + } + + @PostConstruct + void start() { + client = CloudWatchLogsClient.builder().region(region).build(); + ensureLogGroupAndStream(); + backgroundTask = executorService.submit(this::processQueue); + } + + private void processQueue() { + while (running || !queue.isEmpty()) { + List events = new ArrayList<>(); + try { + EventAndTimestamp first = queue.poll(MAX_WAIT_MS, TimeUnit.MILLISECONDS); + + if (first != null) { + events.add(createLogEvent(first)); + List drainedEvents = new ArrayList<>(); + queue.drainTo(drainedEvents, MAX_BATCH_SIZE - 1); + drainedEvents.forEach(event -> events.add(createLogEvent(event))); + } + + if (!events.isEmpty()) { + sendToCloudWatch(events); + } + } catch (Exception e) { + LOGGER.error("Error writing logs to CloudWatch: {}", e.getMessage()); + LOGGER.error("Events not logged: {}", events); + } + } + } + + private InputLogEvent createLogEvent(EventAndTimestamp eventAndTimestamp) { + return InputLogEvent.builder() + .message(eventAndTimestamp.event) + .timestamp(eventAndTimestamp.timestamp) + .build(); + } + + private void sendToCloudWatch(List events) { + events.sort(Comparator.comparingLong(InputLogEvent::timestamp)); + + PutLogEventsRequest.Builder requestBuilder = + PutLogEventsRequest.builder() + .logGroupName(logGroup) + .logStreamName(logStream) + .logEvents(events); + + synchronized (this) { + if (sequenceToken != null) { + requestBuilder.sequenceToken(sequenceToken); + } + + try { + PutLogEventsResponse response = client.putLogEvents(requestBuilder.build()); + sequenceToken = response.nextSequenceToken(); + } catch (InvalidSequenceTokenException e) { + sequenceToken = getSequenceToken(); + requestBuilder.sequenceToken(sequenceToken); + PutLogEventsResponse retryResponse = client.putLogEvents(requestBuilder.build()); + sequenceToken = retryResponse.nextSequenceToken(); + } + } + } + + private void ensureLogGroupAndStream() { + try { + client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); + } catch (ResourceAlreadyExistsException ignored) { + } + + try { + client.createLogStream( + CreateLogStreamRequest.builder().logGroupName(logGroup).logStreamName(logStream).build()); + } catch (ResourceAlreadyExistsException ignored) { + } + + sequenceToken = getSequenceToken(); + } + + private String getSequenceToken() { + DescribeLogStreamsResponse response = + client.describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(logGroup) + .logStreamNamePrefix(logStream) + .build()); + + return response.logStreams().stream() + .filter(s -> logStream.equals(s.logStreamName())) + .map(LogStream::uploadSequenceToken) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + } + + @PreDestroy + void shutdown() { + running = false; + if (backgroundTask != null) { + try { + backgroundTask.get(10, TimeUnit.SECONDS); + } catch (Exception e) { + LOGGER.error("Error waiting for background logging task to finish: {}", e.getMessage()); + } + } + if (client != null) { + client.close(); + } + } + + private record EventAndTimestamp(String event, long timestamp) {} + + // Event overrides below + @Override + public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext callContext) { + try { + Map json = objectMapper.convertValue(event.catalog(), Map.class); + json.put("realm", callContext.getRealmContext().getRealmIdentifier()); + json.put("event_type", event.getClass().getSimpleName()); + queue.add( + new EventAndTimestamp( + objectMapper.writeValueAsString(json), System.currentTimeMillis())); + } catch (JsonProcessingException e) { + LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); + } + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java new file mode 100644 index 0000000000..60e3660996 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.events; + +import java.util.Optional; + +public interface EventListenerConfiguration { + Optional awsCloudwatchlogGroup(); + + Optional awsCloudwatchlogStream(); + + Optional awsCloudwatchRegion(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java index 485766bb24..44cf6eaa13 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.events; +import org.apache.polaris.core.context.CallContext; + /** * Represents an event listener that can respond to notable moments during Polaris's execution. * Event details are documented under the event objects themselves. @@ -55,4 +57,7 @@ public void onBeforeTaskAttempted(BeforeTaskAttemptedEvent event) {} /** {@link AfterTaskAttemptedEvent} */ public void onAfterTaskAttempted(AfterTaskAttemptedEvent event) {} + + /** {@link AfterCatalogCreatedEvent} */ + public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext callContext) {} } diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 48f4d713b9..63c278ada0 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -258,7 +258,8 @@ public String getAuthenticationScheme() { userSecretsManagerFactory, authorizer, callContext, - reservedProperties)); + reservedProperties, + polarisEventListener)); return new TestServices( catalogsApi, From 06eaca48f0961ad1f101c171d2ac13fd073af7f1 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Thu, 26 Jun 2025 19:17:07 -0700 Subject: [PATCH 02/39] cleanup --- .../core/config/FeatureConfiguration.java | 21 ------------------- .../src/main/resources/application.properties | 13 +++--------- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index caf1c5222f..fe265c3072 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -298,25 +298,4 @@ public static void enforceFeatureEnabledOrThrow( + "in their snapshot summary") .defaultValue(false) .buildFeatureConfiguration(); - - public static final FeatureConfiguration AWS_CLOUDWATCH_EVENT_LISTENER_LOG_GROUP = - PolarisConfiguration.builder() - .key("AWS_CLOUDWATCH_EVENT_LISTENER_LOG_GROUP") - .description("The log group you'd like the AWS CloudWatch Event Listener to output to.") - .defaultValue("polaris-log-group") - .buildFeatureConfiguration(); - - public static final FeatureConfiguration AWS_CLOUDWATCH_EVENT_LISTENER_LOG_STREAM = - PolarisConfiguration.builder() - .key("AWS_CLOUDWATCH_EVENT_LISTENER_LOG_STREAM") - .description("The log stream you'd like the AWS CloudWatch Event Listener to output to.") - .defaultValue("polaris-log-stream") - .buildFeatureConfiguration(); - - public static final FeatureConfiguration AWS_CLOUDWATCH_EVENT_LISTENER_REGION = - PolarisConfiguration.builder() - .key("AWS_CLOUDWATCH_EVENT_LISTENER_REGION") - .description("The region where the log group is located for the AWS CloudWatch Event Listener.") - .defaultValue("us-east-1") - .buildFeatureConfiguration(); } diff --git a/runtime/defaults/src/main/resources/application.properties b/runtime/defaults/src/main/resources/application.properties index 508f4e51cb..cbbb037fdb 100644 --- a/runtime/defaults/src/main/resources/application.properties +++ b/runtime/defaults/src/main/resources/application.properties @@ -109,9 +109,7 @@ polaris.realm-context.header-name=Polaris-Realm polaris.realm-context.require-header=false polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false -polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] -polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true -polaris.readiness.ignore-severe-issues=true +polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"] # polaris.features."ENABLE_CATALOG_FEDERATION"=true polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] @@ -120,18 +118,13 @@ polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] # polaris.persistence.type=eclipse-link # polaris.persistence.type=in-memory-atomic -# polaris.persistence.type=in-memory -polaris.persistence.type=relational-jdbc -quarkus.datasource.db-kind=pgsql -quarkus.datasource.jdbc.url=jdbc:postgresql://localhost:5432/POLARIS -quarkus.datasource.username=postgres -quarkus.datasource.password=postgres +polaris.persistence.type=in-memory polaris.secrets-manager.type=in-memory polaris.file-io.type=default -# polaris.event-listener.type=no-op +polaris.event-listener.type=no-op # polaris.event-listener.type=aws-cloudwatch # polaris.event-listener.aws-cloudwatch.log-group=test-group # polaris.event-listener.aws-cloudwatch.log-stream=test-stream From 1515fbb990be5461960f280cc85fc51be160dcc6 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Thu, 26 Jun 2025 19:28:08 -0700 Subject: [PATCH 03/39] spotlessapply --- .../service/quarkus/events/AwsCloudwatchConfig.java | 8 +++++--- .../events/QuarkusPolarisEventListenerConfiguration.java | 3 +-- .../service/events/AwsCloudWatchEventListener.java | 7 +++---- .../service/events/EventListenerConfiguration.java | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java index e0da71140c..d4536f3437 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java @@ -20,7 +20,9 @@ package org.apache.polaris.service.quarkus.events; public interface AwsCloudwatchConfig { - String logGroup(); - String logStream(); - String region(); + String logGroup(); + + String logStream(); + + String region(); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java index b79556ca13..4330d9e4e3 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java @@ -21,9 +21,8 @@ import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; import io.smallrye.config.WithName; -import org.apache.polaris.service.events.EventListenerConfiguration; - import java.util.Optional; +import org.apache.polaris.service.events.EventListenerConfiguration; @StaticInitSafe @ConfigMapping(prefix = "polaris.event-listener") diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index 9af9bca970..bab9f96656 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -36,7 +36,6 @@ import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; - import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,7 +75,8 @@ public class AwsCloudWatchEventListener extends PolarisEventListener { private final Region region; @Inject - public AwsCloudWatchEventListener(EventListenerConfiguration config, ExecutorService executorService) { + public AwsCloudWatchEventListener( + EventListenerConfiguration config, ExecutorService executorService) { this.executorService = executorService; this.logStream = config.awsCloudwatchlogStream().orElse("polaris-cloudwatch-default-stream"); @@ -203,8 +203,7 @@ public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext ca json.put("realm", callContext.getRealmContext().getRealmIdentifier()); json.put("event_type", event.getClass().getSimpleName()); queue.add( - new EventAndTimestamp( - objectMapper.writeValueAsString(json), System.currentTimeMillis())); + new EventAndTimestamp(objectMapper.writeValueAsString(json), System.currentTimeMillis())); } catch (JsonProcessingException e) { LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java index 60e3660996..701e1a3754 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java @@ -22,9 +22,9 @@ import java.util.Optional; public interface EventListenerConfiguration { - Optional awsCloudwatchlogGroup(); + Optional awsCloudwatchlogGroup(); - Optional awsCloudwatchlogStream(); + Optional awsCloudwatchlogStream(); - Optional awsCloudwatchRegion(); + Optional awsCloudwatchRegion(); } From 854501b1bc5340ab7be2e2f24c1b3867cd1f6209 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Sat, 12 Jul 2025 04:55:00 -0700 Subject: [PATCH 04/39] Added unit test with LocalStack --- gradle/libs.versions.toml | 1 + ...rkusPolarisEventListenerConfiguration.java | 3 +- service/common/build.gradle.kts | 2 + .../events/AwsCloudWatchEventListener.java | 58 ++-- .../AwsCloudWatchEventListenerTest.java | 254 ++++++++++++++++++ 5 files changed, 297 insertions(+), 21 deletions(-) create mode 100644 service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 91508b6b59..b4cda0458d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -72,6 +72,7 @@ jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" } javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" } junit-bom = { module = "org.junit:junit-bom", version = "5.13.1" } +localstack = { module = "org.testcontainers:localstack", version = "1.19.7" } logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" } micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.15.1" } microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.2" } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java index 4330d9e4e3..1355d211c2 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java @@ -23,13 +23,14 @@ import io.smallrye.config.WithName; import java.util.Optional; import org.apache.polaris.service.events.EventListenerConfiguration; +import org.apache.polaris.service.events.PolarisEventListener; @StaticInitSafe @ConfigMapping(prefix = "polaris.event-listener") public interface QuarkusPolarisEventListenerConfiguration extends EventListenerConfiguration { /** * The type of the event listener to use. Must be a registered {@link - * org.apache.polaris.service.events.PolarisEventListener} identifier. + * PolarisEventListener} identifier. */ String type(); diff --git a/service/common/build.gradle.kts b/service/common/build.gradle.kts index 21f8886b8a..ab1dbc797b 100644 --- a/service/common/build.gradle.kts +++ b/service/common/build.gradle.kts @@ -95,6 +95,8 @@ dependencies { testImplementation("org.junit.jupiter:junit-jupiter") testImplementation(libs.assertj.core) testImplementation(libs.mockito.core) + testImplementation(libs.localstack) + testImplementation("org.testcontainers:testcontainers") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation(libs.logback.classic) diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index bab9f96656..493fc7a478 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; @@ -76,7 +77,7 @@ public class AwsCloudWatchEventListener extends PolarisEventListener { @Inject public AwsCloudWatchEventListener( - EventListenerConfiguration config, ExecutorService executorService) { + EventListenerConfiguration config, ExecutorService executorService) { this.executorService = executorService; this.logStream = config.awsCloudwatchlogStream().orElse("polaris-cloudwatch-default-stream"); @@ -86,31 +87,44 @@ public AwsCloudWatchEventListener( @PostConstruct void start() { - client = CloudWatchLogsClient.builder().region(region).build(); + this.client = createCloudWatchClient(); ensureLogGroupAndStream(); backgroundTask = executorService.submit(this::processQueue); } + protected CloudWatchLogsClient createCloudWatchClient() { + return CloudWatchLogsClient.builder() + .region(region) + .build(); + } + private void processQueue() { while (running || !queue.isEmpty()) { - List events = new ArrayList<>(); - try { - EventAndTimestamp first = queue.poll(MAX_WAIT_MS, TimeUnit.MILLISECONDS); - - if (first != null) { - events.add(createLogEvent(first)); - List drainedEvents = new ArrayList<>(); - queue.drainTo(drainedEvents, MAX_BATCH_SIZE - 1); - drainedEvents.forEach(event -> events.add(createLogEvent(event))); - } - - if (!events.isEmpty()) { - sendToCloudWatch(events); - } - } catch (Exception e) { - LOGGER.error("Error writing logs to CloudWatch: {}", e.getMessage()); - LOGGER.error("Events not logged: {}", events); + drainQueue(); + } + } + + @VisibleForTesting + public void drainQueue() { + List drainedEvents = new ArrayList<>(); + List transformedEvents = new ArrayList<>(); + try { + EventAndTimestamp first = queue.poll(MAX_WAIT_MS, TimeUnit.MILLISECONDS); + + if (first != null) { + drainedEvents.add(first); + queue.drainTo(drainedEvents, MAX_BATCH_SIZE - 1); + } else { + return; } + + drainedEvents.forEach(event -> transformedEvents.add(createLogEvent(event))); + + sendToCloudWatch(transformedEvents); + } catch (Exception e) { + LOGGER.error("Error writing logs to CloudWatch: {}", e.getMessage()); + LOGGER.error("Events not logged: {}", transformedEvents); + queue.addAll(drainedEvents); } } @@ -195,6 +209,10 @@ void shutdown() { private record EventAndTimestamp(String event, long timestamp) {} + private long getCurrentTimestamp(CallContext callContext) { + return callContext.getPolarisCallContext().getClock().millis(); + } + // Event overrides below @Override public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext callContext) { @@ -203,7 +221,7 @@ public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext ca json.put("realm", callContext.getRealmContext().getRealmIdentifier()); json.put("event_type", event.getClass().getSimpleName()); queue.add( - new EventAndTimestamp(objectMapper.writeValueAsString(json), System.currentTimeMillis())); + new EventAndTimestamp(objectMapper.writeValueAsString(json), getCurrentTimestamp(callContext))); } catch (JsonProcessingException e) { LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); } diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java new file mode 100644 index 0000000000..10c3e937f9 --- /dev/null +++ b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.events.listeners; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.service.events.AfterCatalogCreatedEvent; +import org.apache.polaris.service.events.AwsCloudWatchEventListener; +import org.apache.polaris.service.events.EventListenerConfiguration; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; +import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.OutputLogEvent; + +class AwsCloudWatchEventListenerTest { + private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListenerTest.class); + + private static final DockerImageName LOCALSTACK_IMAGE = DockerImageName.parse("localstack/localstack:3.4"); + + private static final LocalStackContainer localStack = + new LocalStackContainer(LOCALSTACK_IMAGE) + .withServices(LocalStackContainer.Service.CLOUDWATCHLOGS); + + private static final String LOG_GROUP = "test-log-group"; + private static final String LOG_STREAM = "test-log-stream"; + private static final String REALM = "test-realm"; + + @Mock + private EventListenerConfiguration config; + + @Mock + private CallContext callContext; + + @Mock + private RealmContext realmContext; + + @Mock + private PolarisCallContext polarisCallContext; + + private ExecutorService executorService; + private AwsCloudWatchEventListener listener; + private CloudWatchLogsClient cloudWatchLogsClient; + private AutoCloseable mockitoContext; + + @BeforeEach + void setUp() { + localStack.start(); + mockitoContext = MockitoAnnotations.openMocks(this); + executorService = Executors.newSingleThreadExecutor(); + + // Configure the mocks + when(config.awsCloudwatchlogGroup()).thenReturn(Optional.of(LOG_GROUP)); + when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); + when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); + when(callContext.getRealmContext()).thenReturn(realmContext); + when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); + when(polarisCallContext.getClock()).thenReturn(Clock.systemUTC()); + when(realmContext.getRealmIdentifier()).thenReturn(REALM); + + // Create CloudWatch client pointing to LocalStack + cloudWatchLogsClient = CloudWatchLogsClient.builder() + .endpointOverride(localStack.getEndpoint()) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(localStack.getAccessKey(), localStack.getSecretKey()))) + .region(Region.of(localStack.getRegion())) + .build(); + + listener = new AwsCloudWatchEventListener(config, executorService) { + @Override + protected CloudWatchLogsClient createCloudWatchClient() { + return cloudWatchLogsClient; + } + }; + } + + @AfterEach + void tearDown() throws Exception { + if (mockitoContext != null) { + mockitoContext.close(); + } + if (executorService != null) { + executorService.shutdownNow(); + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.warn("ExecutorService did not terminate in time"); + } + } + if (cloudWatchLogsClient != null) { + cloudWatchLogsClient.close(); + } + localStack.stop(); + } + + @Test + void shouldCreateLogGroupAndStream() { + // Start the listener which should create the log group and stream + listener.start(); + + // Verify log group exists + DescribeLogGroupsResponse groups = cloudWatchLogsClient.describeLogGroups(DescribeLogGroupsRequest.builder() + .logGroupNamePrefix(LOG_GROUP) + .build()); + assertThat(groups.logGroups()) + .hasSize(1) + .first() + .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); + + // Verify log stream exists + DescribeLogStreamsResponse streams = cloudWatchLogsClient.describeLogStreams(DescribeLogStreamsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamNamePrefix(LOG_STREAM) + .build()); + assertThat(streams.logStreams()) + .hasSize(1) + .first() + .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + } + + @Test + void shouldSendEventToCloudWatchSingleEventSubmissions() { + listener.start(); + + // Create a test catalog entity + String catalog1Name = "test-catalog1"; + String catalog2Name = "test-catalog2"; + + // Create and send the event + AfterCatalogCreatedEvent event = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); + listener.onAfterCatalogCreated(event, callContext); + + // Wait a bit for the background thread to process + listener.drainQueue(); + + // Verify the event was sent to CloudWatch + GetLogEventsResponse logEvents = cloudWatchLogsClient.getLogEvents(GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()); + + assertThat(logEvents.events()) + .hasSize(1) + .first() + .satisfies(logEvent -> { + String message = logEvent.message(); + assertThat(message).contains(catalog1Name); + assertThat(message).contains(REALM); + assertThat(message).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + }); + + // Redo above procedure to ensure that non-cold-start events can also be submitted + event = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); + listener.onAfterCatalogCreated(event, callContext); + + // Wait a bit for the background thread to process + listener.drainQueue(); + + // Verify the event was sent to CloudWatch + logEvents = cloudWatchLogsClient.getLogEvents(GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()); + + assertThat(logEvents.events()).hasSize(2); + String secondMsg = logEvents.events().get(1).message(); + assertThat(secondMsg).contains(catalog2Name); + assertThat(secondMsg).contains(REALM); + assertThat(secondMsg).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + } + + @Test + void shouldSendEventToCloudWatchBatchEventSubmissions() { + listener.start(); + String catalog1Name = "test-catalog1"; + String catalog2Name = "test-catalog2"; + + AfterCatalogCreatedEvent event1 = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); + AfterCatalogCreatedEvent event2 = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); + + listener.onAfterCatalogCreated(event1, callContext); + listener.onAfterCatalogCreated(event2, callContext); + listener.drainQueue(); + + // Verify the events were sent to CloudWatch + GetLogEventsResponse logEvents = cloudWatchLogsClient.getLogEvents(GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()); + + assertThat(logEvents.events()).hasSize(2); + List sortedEvents = new ArrayList<>(logEvents.events()); + sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); + assertThat(sortedEvents.get(0).message()).contains(catalog1Name); + assertThat(sortedEvents.get(1).message()).contains(catalog2Name); + } + + private Catalog getTestCatalog(String catalogName) { + return PolarisCatalog.builder() + .setName(catalogName) + .setType(Catalog.TypeEnum.INTERNAL) + .setStorageConfigInfo(FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) + .setAllowedLocations(List.of("file://")) + .build()) + .build(); + } +} From c1c94b2e88f51086ecc6b4cf7e4f49fe1865a2b5 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Sat, 12 Jul 2025 05:10:51 -0700 Subject: [PATCH 05/39] typo --- .../service/events/AwsCloudWatchEventListenerTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java index 10c3e937f9..c255edcd69 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.service.events.listeners; +package org.apache.polaris.service.events; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; @@ -210,7 +210,9 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { .build()); assertThat(logEvents.events()).hasSize(2); - String secondMsg = logEvents.events().get(1).message(); + List sortedEvents = new ArrayList<>(logEvents.events()); + sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); + String secondMsg = sortedEvents.get(1).message(); assertThat(secondMsg).contains(catalog2Name); assertThat(secondMsg).contains(REALM); assertThat(secondMsg).contains(AfterCatalogCreatedEvent.class.getSimpleName()); From ab3c5f9ae516f8577b9cc00889ca360d7d8f3e05 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Sat, 12 Jul 2025 05:34:37 -0700 Subject: [PATCH 06/39] spotlessapply --- ...rkusPolarisEventListenerConfiguration.java | 4 +- .../events/AwsCloudWatchEventListener.java | 9 +- .../AwsCloudWatchEventListenerTest.java | 360 +++++++++--------- 3 files changed, 190 insertions(+), 183 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java index 1355d211c2..05bb9fb1b2 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java @@ -29,8 +29,8 @@ @ConfigMapping(prefix = "polaris.event-listener") public interface QuarkusPolarisEventListenerConfiguration extends EventListenerConfiguration { /** - * The type of the event listener to use. Must be a registered {@link - * PolarisEventListener} identifier. + * The type of the event listener to use. Must be a registered {@link PolarisEventListener} + * identifier. */ String type(); diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index 493fc7a478..8aadcccc3d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -77,7 +77,7 @@ public class AwsCloudWatchEventListener extends PolarisEventListener { @Inject public AwsCloudWatchEventListener( - EventListenerConfiguration config, ExecutorService executorService) { + EventListenerConfiguration config, ExecutorService executorService) { this.executorService = executorService; this.logStream = config.awsCloudwatchlogStream().orElse("polaris-cloudwatch-default-stream"); @@ -93,9 +93,7 @@ void start() { } protected CloudWatchLogsClient createCloudWatchClient() { - return CloudWatchLogsClient.builder() - .region(region) - .build(); + return CloudWatchLogsClient.builder().region(region).build(); } private void processQueue() { @@ -221,7 +219,8 @@ public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext ca json.put("realm", callContext.getRealmContext().getRealmIdentifier()); json.put("event_type", event.getClass().getSimpleName()); queue.add( - new EventAndTimestamp(objectMapper.writeValueAsString(json), getCurrentTimestamp(callContext))); + new EventAndTimestamp( + objectMapper.writeValueAsString(json), getCurrentTimestamp(callContext))); } catch (JsonProcessingException e) { LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); } diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java index c255edcd69..53cabd70df 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java @@ -30,7 +30,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; - import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; @@ -38,9 +37,6 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.service.events.AfterCatalogCreatedEvent; -import org.apache.polaris.service.events.AwsCloudWatchEventListener; -import org.apache.polaris.service.events.EventListenerConfiguration; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -63,194 +59,206 @@ import software.amazon.awssdk.services.cloudwatchlogs.model.OutputLogEvent; class AwsCloudWatchEventListenerTest { - private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListenerTest.class); - - private static final DockerImageName LOCALSTACK_IMAGE = DockerImageName.parse("localstack/localstack:3.4"); - - private static final LocalStackContainer localStack = - new LocalStackContainer(LOCALSTACK_IMAGE) - .withServices(LocalStackContainer.Service.CLOUDWATCHLOGS); - - private static final String LOG_GROUP = "test-log-group"; - private static final String LOG_STREAM = "test-log-stream"; - private static final String REALM = "test-realm"; - - @Mock - private EventListenerConfiguration config; - - @Mock - private CallContext callContext; - - @Mock - private RealmContext realmContext; - - @Mock - private PolarisCallContext polarisCallContext; - - private ExecutorService executorService; - private AwsCloudWatchEventListener listener; - private CloudWatchLogsClient cloudWatchLogsClient; - private AutoCloseable mockitoContext; - - @BeforeEach - void setUp() { - localStack.start(); - mockitoContext = MockitoAnnotations.openMocks(this); - executorService = Executors.newSingleThreadExecutor(); - - // Configure the mocks - when(config.awsCloudwatchlogGroup()).thenReturn(Optional.of(LOG_GROUP)); - when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); - when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); - when(callContext.getRealmContext()).thenReturn(realmContext); - when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); - when(polarisCallContext.getClock()).thenReturn(Clock.systemUTC()); - when(realmContext.getRealmIdentifier()).thenReturn(REALM); - - // Create CloudWatch client pointing to LocalStack - cloudWatchLogsClient = CloudWatchLogsClient.builder() - .endpointOverride(localStack.getEndpoint()) - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create(localStack.getAccessKey(), localStack.getSecretKey()))) - .region(Region.of(localStack.getRegion())) - .build(); - - listener = new AwsCloudWatchEventListener(config, executorService) { - @Override - protected CloudWatchLogsClient createCloudWatchClient() { - return cloudWatchLogsClient; - } + private static final Logger LOGGER = + LoggerFactory.getLogger(AwsCloudWatchEventListenerTest.class); + + private static final DockerImageName LOCALSTACK_IMAGE = + DockerImageName.parse("localstack/localstack:3.4"); + + private static final LocalStackContainer localStack = + new LocalStackContainer(LOCALSTACK_IMAGE) + .withServices(LocalStackContainer.Service.CLOUDWATCHLOGS); + + private static final String LOG_GROUP = "test-log-group"; + private static final String LOG_STREAM = "test-log-stream"; + private static final String REALM = "test-realm"; + + @Mock private EventListenerConfiguration config; + + @Mock private CallContext callContext; + + @Mock private RealmContext realmContext; + + @Mock private PolarisCallContext polarisCallContext; + + private ExecutorService executorService; + private AwsCloudWatchEventListener listener; + private CloudWatchLogsClient cloudWatchLogsClient; + private AutoCloseable mockitoContext; + + @BeforeEach + void setUp() { + localStack.start(); + mockitoContext = MockitoAnnotations.openMocks(this); + executorService = Executors.newSingleThreadExecutor(); + + // Configure the mocks + when(config.awsCloudwatchlogGroup()).thenReturn(Optional.of(LOG_GROUP)); + when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); + when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); + when(callContext.getRealmContext()).thenReturn(realmContext); + when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); + when(polarisCallContext.getClock()).thenReturn(Clock.systemUTC()); + when(realmContext.getRealmIdentifier()).thenReturn(REALM); + + // Create CloudWatch client pointing to LocalStack + cloudWatchLogsClient = + CloudWatchLogsClient.builder() + .endpointOverride(localStack.getEndpoint()) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + localStack.getAccessKey(), localStack.getSecretKey()))) + .region(Region.of(localStack.getRegion())) + .build(); + + listener = + new AwsCloudWatchEventListener(config, executorService) { + @Override + protected CloudWatchLogsClient createCloudWatchClient() { + return cloudWatchLogsClient; + } }; - } + } - @AfterEach - void tearDown() throws Exception { - if (mockitoContext != null) { - mockitoContext.close(); - } - if (executorService != null) { - executorService.shutdownNow(); - if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { - LOGGER.warn("ExecutorService did not terminate in time"); - } - } - if (cloudWatchLogsClient != null) { - cloudWatchLogsClient.close(); - } - localStack.stop(); + @AfterEach + void tearDown() throws Exception { + if (mockitoContext != null) { + mockitoContext.close(); } - - @Test - void shouldCreateLogGroupAndStream() { - // Start the listener which should create the log group and stream - listener.start(); - - // Verify log group exists - DescribeLogGroupsResponse groups = cloudWatchLogsClient.describeLogGroups(DescribeLogGroupsRequest.builder() - .logGroupNamePrefix(LOG_GROUP) - .build()); - assertThat(groups.logGroups()) - .hasSize(1) - .first() - .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); - - // Verify log stream exists - DescribeLogStreamsResponse streams = cloudWatchLogsClient.describeLogStreams(DescribeLogStreamsRequest.builder() + if (executorService != null) { + executorService.shutdownNow(); + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.warn("ExecutorService did not terminate in time"); + } + } + if (cloudWatchLogsClient != null) { + cloudWatchLogsClient.close(); + } + localStack.stop(); + } + + @Test + void shouldCreateLogGroupAndStream() { + // Start the listener which should create the log group and stream + listener.start(); + + // Verify log group exists + DescribeLogGroupsResponse groups = + cloudWatchLogsClient.describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()); + assertThat(groups.logGroups()) + .hasSize(1) + .first() + .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); + + // Verify log stream exists + DescribeLogStreamsResponse streams = + cloudWatchLogsClient.describeLogStreams( + DescribeLogStreamsRequest.builder() .logGroupName(LOG_GROUP) .logStreamNamePrefix(LOG_STREAM) .build()); - assertThat(streams.logStreams()) - .hasSize(1) - .first() - .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); - } - - @Test - void shouldSendEventToCloudWatchSingleEventSubmissions() { - listener.start(); - - // Create a test catalog entity - String catalog1Name = "test-catalog1"; - String catalog2Name = "test-catalog2"; - - // Create and send the event - AfterCatalogCreatedEvent event = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); - listener.onAfterCatalogCreated(event, callContext); - - // Wait a bit for the background thread to process - listener.drainQueue(); - - // Verify the event was sent to CloudWatch - GetLogEventsResponse logEvents = cloudWatchLogsClient.getLogEvents(GetLogEventsRequest.builder() + assertThat(streams.logStreams()) + .hasSize(1) + .first() + .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + } + + @Test + void shouldSendEventToCloudWatchSingleEventSubmissions() { + listener.start(); + + // Create a test catalog entity + String catalog1Name = "test-catalog1"; + String catalog2Name = "test-catalog2"; + + // Create and send the event + AfterCatalogCreatedEvent event = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); + listener.onAfterCatalogCreated(event, callContext); + + // Wait a bit for the background thread to process + listener.drainQueue(); + + // Verify the event was sent to CloudWatch + GetLogEventsResponse logEvents = + cloudWatchLogsClient.getLogEvents( + GetLogEventsRequest.builder() .logGroupName(LOG_GROUP) .logStreamName(LOG_STREAM) .build()); - assertThat(logEvents.events()) - .hasSize(1) - .first() - .satisfies(logEvent -> { - String message = logEvent.message(); - assertThat(message).contains(catalog1Name); - assertThat(message).contains(REALM); - assertThat(message).contains(AfterCatalogCreatedEvent.class.getSimpleName()); - }); - - // Redo above procedure to ensure that non-cold-start events can also be submitted - event = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); - listener.onAfterCatalogCreated(event, callContext); - - // Wait a bit for the background thread to process - listener.drainQueue(); - - // Verify the event was sent to CloudWatch - logEvents = cloudWatchLogsClient.getLogEvents(GetLogEventsRequest.builder() + assertThat(logEvents.events()) + .hasSize(1) + .first() + .satisfies( + logEvent -> { + String message = logEvent.message(); + assertThat(message).contains(catalog1Name); + assertThat(message).contains(REALM); + assertThat(message).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + }); + + // Redo above procedure to ensure that non-cold-start events can also be submitted + event = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); + listener.onAfterCatalogCreated(event, callContext); + + // Wait a bit for the background thread to process + listener.drainQueue(); + + // Verify the event was sent to CloudWatch + logEvents = + cloudWatchLogsClient.getLogEvents( + GetLogEventsRequest.builder() .logGroupName(LOG_GROUP) .logStreamName(LOG_STREAM) .build()); - assertThat(logEvents.events()).hasSize(2); - List sortedEvents = new ArrayList<>(logEvents.events()); - sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); - String secondMsg = sortedEvents.get(1).message(); - assertThat(secondMsg).contains(catalog2Name); - assertThat(secondMsg).contains(REALM); - assertThat(secondMsg).contains(AfterCatalogCreatedEvent.class.getSimpleName()); - } - - @Test - void shouldSendEventToCloudWatchBatchEventSubmissions() { - listener.start(); - String catalog1Name = "test-catalog1"; - String catalog2Name = "test-catalog2"; - - AfterCatalogCreatedEvent event1 = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); - AfterCatalogCreatedEvent event2 = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); - - listener.onAfterCatalogCreated(event1, callContext); - listener.onAfterCatalogCreated(event2, callContext); - listener.drainQueue(); - - // Verify the events were sent to CloudWatch - GetLogEventsResponse logEvents = cloudWatchLogsClient.getLogEvents(GetLogEventsRequest.builder() + assertThat(logEvents.events()).hasSize(2); + List sortedEvents = new ArrayList<>(logEvents.events()); + sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); + String secondMsg = sortedEvents.get(1).message(); + assertThat(secondMsg).contains(catalog2Name); + assertThat(secondMsg).contains(REALM); + assertThat(secondMsg).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + } + + @Test + void shouldSendEventToCloudWatchBatchEventSubmissions() { + listener.start(); + String catalog1Name = "test-catalog1"; + String catalog2Name = "test-catalog2"; + + AfterCatalogCreatedEvent event1 = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); + AfterCatalogCreatedEvent event2 = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); + + listener.onAfterCatalogCreated(event1, callContext); + listener.onAfterCatalogCreated(event2, callContext); + listener.drainQueue(); + + // Verify the events were sent to CloudWatch + GetLogEventsResponse logEvents = + cloudWatchLogsClient.getLogEvents( + GetLogEventsRequest.builder() .logGroupName(LOG_GROUP) .logStreamName(LOG_STREAM) .build()); - assertThat(logEvents.events()).hasSize(2); - List sortedEvents = new ArrayList<>(logEvents.events()); - sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); - assertThat(sortedEvents.get(0).message()).contains(catalog1Name); - assertThat(sortedEvents.get(1).message()).contains(catalog2Name); - } - - private Catalog getTestCatalog(String catalogName) { - return PolarisCatalog.builder() - .setName(catalogName) - .setType(Catalog.TypeEnum.INTERNAL) - .setStorageConfigInfo(FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://")) - .build()) - .build(); - } + assertThat(logEvents.events()).hasSize(2); + List sortedEvents = new ArrayList<>(logEvents.events()); + sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); + assertThat(sortedEvents.get(0).message()).contains(catalog1Name); + assertThat(sortedEvents.get(1).message()).contains(catalog2Name); + } + + private Catalog getTestCatalog(String catalogName) { + return PolarisCatalog.builder() + .setName(catalogName) + .setType(Catalog.TypeEnum.INTERNAL) + .setStorageConfigInfo( + FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) + .setAllowedLocations(List.of("file://")) + .build()) + .build(); + } } From a6411366b992f04b8f09debd566191de9299a18f Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Sat, 12 Jul 2025 05:45:12 -0700 Subject: [PATCH 07/39] recompile from main --- .../polaris/service/admin/PolarisServiceImplTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/service/common/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java b/service/common/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java index fab1d7da41..598c9f80bf 100644 --- a/service/common/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java @@ -42,6 +42,7 @@ import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.config.ReservedProperties; +import org.apache.polaris.service.events.PolarisEventListener; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -57,6 +58,7 @@ public class PolarisServiceImplTest { private PolarisCallContext polarisCallContext; private PolarisConfigurationStore configurationStore; private RealmContext realmContext; + private PolarisEventListener polarisEventListener; private PolarisServiceImpl polarisService; @@ -71,6 +73,7 @@ void setUp() { polarisCallContext = Mockito.mock(PolarisCallContext.class); configurationStore = Mockito.mock(PolarisConfigurationStore.class); realmContext = Mockito.mock(RealmContext.class); + polarisEventListener = Mockito.mock(PolarisEventListener.class); when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); when(callContext.getRealmContext()).thenReturn(realmContext); @@ -89,7 +92,8 @@ void setUp() { userSecretsManagerFactory, polarisAuthorizer, callContext, - reservedProperties); + reservedProperties, + polarisEventListener); } @Test From 5a355d18ecfc44d594ad13eaff34ab5afd31736e Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 15 Jul 2025 21:57:28 -0700 Subject: [PATCH 08/39] first revision change, based on review from @eric-maynard --- .../src/main/resources/application.properties | 6 +- .../events/AwsCloudWatchEventListener.java | 82 +++++++++++++------ .../AwsCloudWatchEventListenerTest.java | 20 ++++- 3 files changed, 75 insertions(+), 33 deletions(-) diff --git a/runtime/defaults/src/main/resources/application.properties b/runtime/defaults/src/main/resources/application.properties index d6666ffe29..0130c0134e 100644 --- a/runtime/defaults/src/main/resources/application.properties +++ b/runtime/defaults/src/main/resources/application.properties @@ -125,11 +125,7 @@ polaris.secrets-manager.type=in-memory polaris.file-io.type=default -polaris.event-listener.type=no-op -# polaris.event-listener.type=aws-cloudwatch -# polaris.event-listener.aws-cloudwatch.log-group=test-group -# polaris.event-listener.aws-cloudwatch.log-stream=test-stream -# polaris.event-listener.aws-cloudwatch.region=us-west-2 +polaris.event-listener.type=aws-cloudwatch polaris.log.request-id-header-name=Polaris-Request-Id # polaris.log.mdc.aid=polaris diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index 8aadcccc3d..8bbc04c7d1 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -20,6 +20,7 @@ package org.apache.polaris.service.events; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; @@ -40,18 +41,22 @@ import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.DataAlreadyAcceptedException; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.InputLogEvent; +import software.amazon.awssdk.services.cloudwatchlogs.model.InvalidParameterException; import software.amazon.awssdk.services.cloudwatchlogs.model.InvalidSequenceTokenException; import software.amazon.awssdk.services.cloudwatchlogs.model.LogStream; import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.ResourceAlreadyExistsException; +import software.amazon.awssdk.services.cloudwatchlogs.model.UnrecognizedClientException; @ApplicationScoped @Identifier("aws-cloudwatch") @@ -60,8 +65,12 @@ public class AwsCloudWatchEventListener extends PolarisEventListener { private final ObjectMapper objectMapper = new ObjectMapper(); private static final int MAX_BATCH_SIZE = 10_000; private static final int MAX_WAIT_MS = 5000; + private static final String DEFAULT_LOG_STREAM_NAME = "polaris-cloudwatch-default-stream"; + private static final String DEFAULT_LOG_GROUP_NAME = "polaris-cloudwatch-default-group"; + private static final String DEFAULT_REGION = "us-east-1"; - private final BlockingQueue queue = new LinkedBlockingQueue<>(); + + private final BlockingQueue queue = new LinkedBlockingQueue<>(); private CloudWatchLogsClient client; private volatile String sequenceToken; @@ -80,9 +89,9 @@ public AwsCloudWatchEventListener( EventListenerConfiguration config, ExecutorService executorService) { this.executorService = executorService; - this.logStream = config.awsCloudwatchlogStream().orElse("polaris-cloudwatch-default-stream"); - this.logGroup = config.awsCloudwatchlogGroup().orElse("polaris-cloudwatch-default-group"); - this.region = Region.of(config.awsCloudwatchRegion().orElse("us-east-1")); + this.logStream = config.awsCloudwatchlogStream().orElse(DEFAULT_LOG_STREAM_NAME); + this.logGroup = config.awsCloudwatchlogGroup().orElse(DEFAULT_LOG_GROUP_NAME); + this.region = Region.of(config.awsCloudwatchRegion().orElse(DEFAULT_REGION)); } @PostConstruct @@ -97,17 +106,17 @@ protected CloudWatchLogsClient createCloudWatchClient() { } private void processQueue() { + List drainedEvents = new ArrayList<>(); + List transformedEvents = new ArrayList<>(); while (running || !queue.isEmpty()) { - drainQueue(); + drainQueue(drainedEvents, transformedEvents); } } @VisibleForTesting - public void drainQueue() { - List drainedEvents = new ArrayList<>(); - List transformedEvents = new ArrayList<>(); + public void drainQueue(List drainedEvents, List transformedEvents) { try { - EventAndTimestamp first = queue.poll(MAX_WAIT_MS, TimeUnit.MILLISECONDS); + EventWithTimestamp first = queue.poll(MAX_WAIT_MS, TimeUnit.MILLISECONDS); if (first != null) { drainedEvents.add(first); @@ -119,17 +128,31 @@ public void drainQueue() { drainedEvents.forEach(event -> transformedEvents.add(createLogEvent(event))); sendToCloudWatch(transformedEvents); - } catch (Exception e) { - LOGGER.error("Error writing logs to CloudWatch: {}", e.getMessage()); - LOGGER.error("Events not logged: {}", transformedEvents); + } catch (InterruptedException e) { + if (!queue.isEmpty()) { + LOGGER.debug("Interrupted while waiting for queue to drain", e); + queue.addAll(drainedEvents); + } + } catch (DataAlreadyAcceptedException e) { + LOGGER.debug("Data already accepted: {}", e.getMessage()); + } catch (RuntimeException e) { + if (e instanceof SdkClientException || e instanceof InvalidParameterException || e instanceof UnrecognizedClientException) { + LOGGER.error("Error writing logs to CloudWatch - client-side error. Please adjust Polaris configurations: {}", e.getMessage()); + } else { + LOGGER.error("Error writing logs to CloudWatch - server-side error: {}", e.getMessage()); + } + LOGGER.error("Number of dropped events: {}", transformedEvents.size()); + LOGGER.debug("Events not logged: {}", transformedEvents); queue.addAll(drainedEvents); } + drainedEvents.clear(); + transformedEvents.clear(); } - private InputLogEvent createLogEvent(EventAndTimestamp eventAndTimestamp) { + private InputLogEvent createLogEvent(EventWithTimestamp eventWithTimestamp) { return InputLogEvent.builder() - .message(eventAndTimestamp.event) - .timestamp(eventAndTimestamp.timestamp) + .message(eventWithTimestamp.event) + .timestamp(eventWithTimestamp.timestamp) .build(); } @@ -148,27 +171,32 @@ private void sendToCloudWatch(List events) { } try { - PutLogEventsResponse response = client.putLogEvents(requestBuilder.build()); - sequenceToken = response.nextSequenceToken(); + executePutLogEvents(requestBuilder); } catch (InvalidSequenceTokenException e) { sequenceToken = getSequenceToken(); requestBuilder.sequenceToken(sequenceToken); - PutLogEventsResponse retryResponse = client.putLogEvents(requestBuilder.build()); - sequenceToken = retryResponse.nextSequenceToken(); + executePutLogEvents(requestBuilder); } } } + private void executePutLogEvents(PutLogEventsRequest.Builder requestBuilder) { + PutLogEventsResponse response = client.putLogEvents(requestBuilder.build()); + sequenceToken = response.nextSequenceToken(); + } + private void ensureLogGroupAndStream() { try { client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); } catch (ResourceAlreadyExistsException ignored) { + LOGGER.debug("Log group {} already exists", logGroup); } try { client.createLogStream( CreateLogStreamRequest.builder().logGroupName(logGroup).logStreamName(logStream).build()); } catch (ResourceAlreadyExistsException ignored) { + LOGGER.debug("Log stream {} already exists", logStream); } sequenceToken = getSequenceToken(); @@ -205,24 +233,28 @@ void shutdown() { } } - private record EventAndTimestamp(String event, long timestamp) {} + record EventWithTimestamp(String event, long timestamp) {} private long getCurrentTimestamp(CallContext callContext) { return callContext.getPolarisCallContext().getClock().millis(); } // Event overrides below - @Override - public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext callContext) { + private void queueEvent(PolarisEvent event, CallContext callContext) { try { - Map json = objectMapper.convertValue(event.catalog(), Map.class); + Map json = objectMapper.convertValue(event, new TypeReference<>() {}); json.put("realm", callContext.getRealmContext().getRealmIdentifier()); json.put("event_type", event.getClass().getSimpleName()); queue.add( - new EventAndTimestamp( - objectMapper.writeValueAsString(json), getCurrentTimestamp(callContext))); + new EventWithTimestamp( + objectMapper.writeValueAsString(json), getCurrentTimestamp(callContext))); } catch (JsonProcessingException e) { LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); } } + + @Override + public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext callContext) { + queueEvent(event, callContext); + } } diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java index 53cabd70df..db07eb795a 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java @@ -56,6 +56,7 @@ import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.InputLogEvent; import software.amazon.awssdk.services.cloudwatchlogs.model.OutputLogEvent; class AwsCloudWatchEventListenerTest { @@ -178,7 +179,9 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { listener.onAfterCatalogCreated(event, callContext); // Wait a bit for the background thread to process - listener.drainQueue(); + List drainedEvents = new ArrayList<>(); + List transformedEvents = new ArrayList<>(); + listener.drainQueue(drainedEvents, transformedEvents); // Verify the event was sent to CloudWatch GetLogEventsResponse logEvents = @@ -199,12 +202,15 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { assertThat(message).contains(AfterCatalogCreatedEvent.class.getSimpleName()); }); + assertThat(transformedEvents).isEmpty(); + assertThat(drainedEvents).isEmpty(); + // Redo above procedure to ensure that non-cold-start events can also be submitted event = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); listener.onAfterCatalogCreated(event, callContext); // Wait a bit for the background thread to process - listener.drainQueue(); + listener.drainQueue(drainedEvents, transformedEvents); // Verify the event was sent to CloudWatch logEvents = @@ -221,6 +227,9 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { assertThat(secondMsg).contains(catalog2Name); assertThat(secondMsg).contains(REALM); assertThat(secondMsg).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + + assertThat(transformedEvents).isEmpty(); + assertThat(drainedEvents).isEmpty(); } @Test @@ -234,7 +243,9 @@ void shouldSendEventToCloudWatchBatchEventSubmissions() { listener.onAfterCatalogCreated(event1, callContext); listener.onAfterCatalogCreated(event2, callContext); - listener.drainQueue(); + List drainedEvents = new ArrayList<>(); + List transformedEvents = new ArrayList<>(); + listener.drainQueue(drainedEvents, transformedEvents); // Verify the events were sent to CloudWatch GetLogEventsResponse logEvents = @@ -249,6 +260,9 @@ void shouldSendEventToCloudWatchBatchEventSubmissions() { sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); assertThat(sortedEvents.get(0).message()).contains(catalog1Name); assertThat(sortedEvents.get(1).message()).contains(catalog2Name); + + assertThat(transformedEvents).isEmpty(); + assertThat(drainedEvents).isEmpty(); } private Catalog getTestCatalog(String catalogName) { From 04c310ade3c78681b5e5967633557d0b4f739b87 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 15 Jul 2025 22:36:25 -0700 Subject: [PATCH 09/39] spotlessapply --- .../src/main/resources/application.properties | 2 +- .../events/AwsCloudWatchEventListener.java | 16 ++++++++++------ .../service/admin/PolarisServiceImplTest.java | 7 ------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/runtime/defaults/src/main/resources/application.properties b/runtime/defaults/src/main/resources/application.properties index 0130c0134e..7e9fbadac3 100644 --- a/runtime/defaults/src/main/resources/application.properties +++ b/runtime/defaults/src/main/resources/application.properties @@ -125,7 +125,7 @@ polaris.secrets-manager.type=in-memory polaris.file-io.type=default -polaris.event-listener.type=aws-cloudwatch +polaris.event-listener.type=no-op polaris.log.request-id-header-name=Polaris-Request-Id # polaris.log.mdc.aid=polaris diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index 8bbc04c7d1..c88303b4bd 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -69,7 +69,6 @@ public class AwsCloudWatchEventListener extends PolarisEventListener { private static final String DEFAULT_LOG_GROUP_NAME = "polaris-cloudwatch-default-group"; private static final String DEFAULT_REGION = "us-east-1"; - private final BlockingQueue queue = new LinkedBlockingQueue<>(); private CloudWatchLogsClient client; private volatile String sequenceToken; @@ -114,7 +113,8 @@ private void processQueue() { } @VisibleForTesting - public void drainQueue(List drainedEvents, List transformedEvents) { + public void drainQueue( + List drainedEvents, List transformedEvents) { try { EventWithTimestamp first = queue.poll(MAX_WAIT_MS, TimeUnit.MILLISECONDS); @@ -136,8 +136,12 @@ public void drainQueue(List drainedEvents, List Date: Wed, 16 Jul 2025 18:16:57 -0700 Subject: [PATCH 10/39] injected securitycontext and callcontext --- .../service/admin/PolarisServiceImpl.java | 3 +-- .../events/AwsCloudWatchEventListener.java | 16 +++++++++++++--- .../service/events/PolarisEventListener.java | 2 +- .../events/AwsCloudWatchEventListenerTest.java | 18 ++++-------------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 49165dae70..e256297b9a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -154,8 +154,7 @@ public Response createCatalog( validateExternalCatalog(catalog); Catalog newCatalog = new CatalogEntity(adminService.createCatalog(request)).asCatalog(); LOGGER.info("Created new catalog {}", newCatalog); - polarisEventListener.onAfterCatalogCreated( - new AfterCatalogCreatedEvent(newCatalog), callContext); + polarisEventListener.onAfterCatalogCreated(new AfterCatalogCreatedEvent(newCatalog)); return Response.status(Response.Status.CREATED).build(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index c88303b4bd..5901ddc395 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -38,6 +38,9 @@ import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; + +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.SecurityContext; import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,6 +86,12 @@ public class AwsCloudWatchEventListener extends PolarisEventListener { private final String logStream; private final Region region; + @Inject + CallContext callContext; + + @Context + SecurityContext securityContext; + @Inject public AwsCloudWatchEventListener( EventListenerConfiguration config, ExecutorService executorService) { @@ -244,11 +253,12 @@ private long getCurrentTimestamp(CallContext callContext) { } // Event overrides below - private void queueEvent(PolarisEvent event, CallContext callContext) { + private void queueEvent(PolarisEvent event) { try { Map json = objectMapper.convertValue(event, new TypeReference<>() {}); json.put("realm", callContext.getRealmContext().getRealmIdentifier()); json.put("event_type", event.getClass().getSimpleName()); + json.put("principal", securityContext.getUserPrincipal().getName()); queue.add( new EventWithTimestamp( objectMapper.writeValueAsString(json), getCurrentTimestamp(callContext))); @@ -258,7 +268,7 @@ private void queueEvent(PolarisEvent event, CallContext callContext) { } @Override - public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext callContext) { - queueEvent(event, callContext); + public void onAfterCatalogCreated(AfterCatalogCreatedEvent event) { + queueEvent(event); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java index 44cf6eaa13..f19e4c2ea8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java @@ -59,5 +59,5 @@ public void onBeforeTaskAttempted(BeforeTaskAttemptedEvent event) {} public void onAfterTaskAttempted(AfterTaskAttemptedEvent event) {} /** {@link AfterCatalogCreatedEvent} */ - public void onAfterCatalogCreated(AfterCatalogCreatedEvent event, CallContext callContext) {} + public void onAfterCatalogCreated(AfterCatalogCreatedEvent event) {} } diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java index db07eb795a..d9613cc0a0 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java @@ -76,12 +76,6 @@ class AwsCloudWatchEventListenerTest { @Mock private EventListenerConfiguration config; - @Mock private CallContext callContext; - - @Mock private RealmContext realmContext; - - @Mock private PolarisCallContext polarisCallContext; - private ExecutorService executorService; private AwsCloudWatchEventListener listener; private CloudWatchLogsClient cloudWatchLogsClient; @@ -97,10 +91,6 @@ void setUp() { when(config.awsCloudwatchlogGroup()).thenReturn(Optional.of(LOG_GROUP)); when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); - when(callContext.getRealmContext()).thenReturn(realmContext); - when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); - when(polarisCallContext.getClock()).thenReturn(Clock.systemUTC()); - when(realmContext.getRealmIdentifier()).thenReturn(REALM); // Create CloudWatch client pointing to LocalStack cloudWatchLogsClient = @@ -176,7 +166,7 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { // Create and send the event AfterCatalogCreatedEvent event = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); - listener.onAfterCatalogCreated(event, callContext); + listener.onAfterCatalogCreated(event); // Wait a bit for the background thread to process List drainedEvents = new ArrayList<>(); @@ -207,7 +197,7 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { // Redo above procedure to ensure that non-cold-start events can also be submitted event = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); - listener.onAfterCatalogCreated(event, callContext); + listener.onAfterCatalogCreated(event); // Wait a bit for the background thread to process listener.drainQueue(drainedEvents, transformedEvents); @@ -241,8 +231,8 @@ void shouldSendEventToCloudWatchBatchEventSubmissions() { AfterCatalogCreatedEvent event1 = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); AfterCatalogCreatedEvent event2 = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); - listener.onAfterCatalogCreated(event1, callContext); - listener.onAfterCatalogCreated(event2, callContext); + listener.onAfterCatalogCreated(event1); + listener.onAfterCatalogCreated(event2); List drainedEvents = new ArrayList<>(); List transformedEvents = new ArrayList<>(); listener.drainQueue(drainedEvents, transformedEvents); From cc715ad861c43f1b0f731abe1eeb2f186572a0ac Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Wed, 16 Jul 2025 18:17:54 -0700 Subject: [PATCH 11/39] todo --- .../polaris/service/events/AwsCloudWatchEventListener.java | 1 + 1 file changed, 1 insertion(+) diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index 5901ddc395..9fcd45550e 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -259,6 +259,7 @@ private void queueEvent(PolarisEvent event) { json.put("realm", callContext.getRealmContext().getRealmIdentifier()); json.put("event_type", event.getClass().getSimpleName()); json.put("principal", securityContext.getUserPrincipal().getName()); + // TODO: Add request ID when it is available queue.add( new EventWithTimestamp( objectMapper.writeValueAsString(json), getCurrentTimestamp(callContext))); From 518aaaa9a8475206c20b3281e285418bb27a24b6 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Thu, 17 Jul 2025 16:19:05 -0700 Subject: [PATCH 12/39] modify test --- .../AwsCloudWatchEventListenerTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java index d9613cc0a0..419a1dc776 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; +import java.security.Principal; import java.time.Clock; import java.util.ArrayList; import java.util.Comparator; @@ -30,6 +31,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; + +import jakarta.ws.rs.core.SecurityContext; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; @@ -41,6 +44,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +77,7 @@ class AwsCloudWatchEventListenerTest { private static final String LOG_GROUP = "test-log-group"; private static final String LOG_STREAM = "test-log-stream"; private static final String REALM = "test-realm"; + private static final String TEST_USER = "test-user"; @Mock private EventListenerConfiguration config; @@ -92,6 +97,7 @@ void setUp() { when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); + // Create CloudWatch client pointing to LocalStack cloudWatchLogsClient = CloudWatchLogsClient.builder() @@ -110,6 +116,20 @@ protected CloudWatchLogsClient createCloudWatchClient() { return cloudWatchLogsClient; } }; + + CallContext callContext = Mockito.mock(CallContext.class); + PolarisCallContext polarisCallContext = Mockito.mock(PolarisCallContext.class); + RealmContext realmContext = Mockito.mock(RealmContext.class); + SecurityContext securityContext = Mockito.mock(SecurityContext.class); + Principal principal = Mockito.mock(Principal.class); + when(callContext.getRealmContext()).thenReturn(realmContext); + when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); + when(polarisCallContext.getClock()).thenReturn(Clock.systemUTC()); + when(realmContext.getRealmIdentifier()).thenReturn(REALM); + when(securityContext.getUserPrincipal()).thenReturn(principal); + when(principal.getName()).thenReturn(TEST_USER); + listener.callContext = callContext; + listener.securityContext = securityContext; } @AfterEach @@ -190,6 +210,7 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { assertThat(message).contains(catalog1Name); assertThat(message).contains(REALM); assertThat(message).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + assertThat(message).contains(TEST_USER); }); assertThat(transformedEvents).isEmpty(); @@ -217,6 +238,7 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { assertThat(secondMsg).contains(catalog2Name); assertThat(secondMsg).contains(REALM); assertThat(secondMsg).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + assertThat(secondMsg).contains(TEST_USER); assertThat(transformedEvents).isEmpty(); assertThat(drainedEvents).isEmpty(); From 87582552b0062ba93b32e9aa49775d3e3760fcd3 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Sat, 19 Jul 2025 21:42:36 -0700 Subject: [PATCH 13/39] first draft of revision --- .../service/admin/PolarisServiceImpl.java | 2 - .../events/AwsCloudWatchEventListener.java | 225 +++++++++--------- .../service/events/PolarisEventListener.java | 5 - .../AwsCloudWatchEventListenerTest.java | 133 +++++------ .../in-dev/unreleased/configuration.md | 4 + 5 files changed, 167 insertions(+), 202 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index e256297b9a..643579c069 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -79,7 +79,6 @@ import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.config.ReservedProperties; -import org.apache.polaris.service.events.AfterCatalogCreatedEvent; import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.types.PolicyIdentifier; import org.slf4j.Logger; @@ -154,7 +153,6 @@ public Response createCatalog( validateExternalCatalog(catalog); Catalog newCatalog = new CatalogEntity(adminService.createCatalog(request)).asCatalog(); LOGGER.info("Created new catalog {}", newCatalog); - polarisEventListener.onAfterCatalogCreated(new AfterCatalogCreatedEvent(newCatalog)); return Response.status(Response.Status.CREATED).build(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index 9fcd45550e..ecd0c93697 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -20,7 +20,6 @@ package org.apache.polaris.service.events; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; @@ -28,19 +27,14 @@ import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import java.util.ArrayList; -import java.util.Comparator; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.SecurityContext; +import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import jakarta.ws.rs.core.Context; -import jakarta.ws.rs.core.SecurityContext; import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,31 +60,25 @@ public class AwsCloudWatchEventListener extends PolarisEventListener { private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListener.class); private final ObjectMapper objectMapper = new ObjectMapper(); - private static final int MAX_BATCH_SIZE = 10_000; - private static final int MAX_WAIT_MS = 5000; + private final ConcurrentHashMap, EventWithRetry> futures = new ConcurrentHashMap<>(); private static final String DEFAULT_LOG_STREAM_NAME = "polaris-cloudwatch-default-stream"; private static final String DEFAULT_LOG_GROUP_NAME = "polaris-cloudwatch-default-group"; private static final String DEFAULT_REGION = "us-east-1"; - private final BlockingQueue queue = new LinkedBlockingQueue<>(); + record EventWithRetry(InputLogEvent inputLogEvent, int retryCount) {} + private CloudWatchLogsClient client; private volatile String sequenceToken; - private volatile boolean running = true; - ExecutorService executorService; - private Future backgroundTask; - private final String logGroup; private final String logStream; private final Region region; - @Inject - CallContext callContext; + @Inject CallContext callContext; - @Context - SecurityContext securityContext; + @Context SecurityContext securityContext; @Inject public AwsCloudWatchEventListener( @@ -106,42 +94,64 @@ public AwsCloudWatchEventListener( void start() { this.client = createCloudWatchClient(); ensureLogGroupAndStream(); - backgroundTask = executorService.submit(this::processQueue); } protected CloudWatchLogsClient createCloudWatchClient() { return CloudWatchLogsClient.builder().region(region).build(); } - private void processQueue() { - List drainedEvents = new ArrayList<>(); - List transformedEvents = new ArrayList<>(); - while (running || !queue.isEmpty()) { - drainQueue(drainedEvents, transformedEvents); + private void ensureLogGroupAndStream() { + try { + client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); + } catch (ResourceAlreadyExistsException ignored) { + LOGGER.debug("Log group {} already exists", logGroup); } - } - @VisibleForTesting - public void drainQueue( - List drainedEvents, List transformedEvents) { try { - EventWithTimestamp first = queue.poll(MAX_WAIT_MS, TimeUnit.MILLISECONDS); + client.createLogStream( + CreateLogStreamRequest.builder().logGroupName(logGroup).logStreamName(logStream).build()); + } catch (ResourceAlreadyExistsException ignored) { + LOGGER.debug("Log stream {} already exists", logStream); + } - if (first != null) { - drainedEvents.add(first); - queue.drainTo(drainedEvents, MAX_BATCH_SIZE - 1); - } else { - return; - } + sequenceToken = getSequenceToken(); + } - drainedEvents.forEach(event -> transformedEvents.add(createLogEvent(event))); + private String getSequenceToken() { + DescribeLogStreamsResponse response = + client.describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(logGroup) + .logStreamNamePrefix(logStream) + .build()); + + return response.logStreams().stream() + .filter(s -> logStream.equals(s.logStreamName())) + .map(LogStream::uploadSequenceToken) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + } - sendToCloudWatch(transformedEvents); - } catch (InterruptedException e) { - if (!queue.isEmpty()) { - LOGGER.debug("Interrupted while waiting for queue to drain", e); - queue.addAll(drainedEvents); + @PreDestroy + void shutdown() { + for (Future future : futures.keySet()) { + if (!future.state().equals(Future.State.SUCCESS)) { + LOGGER.debug( + "Event not emitted to AWS CloudWatch due to being in state {}: {}", + future.state(), + futures.get(future).inputLogEvent); + future.cancel(true); } + } + if (client != null) { + client.close(); + } + } + + private void sendAndHandleCloudWatchCall(InputLogEvent event) { + try { + sendToCloudWatch(event); } catch (DataAlreadyAcceptedException e) { LOGGER.debug("Data already accepted: {}", e.getMessage()); } catch (RuntimeException e) { @@ -154,29 +164,22 @@ public void drainQueue( } else { LOGGER.error("Error writing logs to CloudWatch - server-side error: {}", e.getMessage()); } - LOGGER.error("Number of dropped events: {}", transformedEvents.size()); - LOGGER.debug("Events not logged: {}", transformedEvents); - queue.addAll(drainedEvents); + throw e; + } finally { + try { + reapFuturesMap(); + } catch (Exception e) { + LOGGER.debug("Futures map could not be reaped: {}", e.getMessage()); + } } - drainedEvents.clear(); - transformedEvents.clear(); } - private InputLogEvent createLogEvent(EventWithTimestamp eventWithTimestamp) { - return InputLogEvent.builder() - .message(eventWithTimestamp.event) - .timestamp(eventWithTimestamp.timestamp) - .build(); - } - - private void sendToCloudWatch(List events) { - events.sort(Comparator.comparingLong(InputLogEvent::timestamp)); - + private void sendToCloudWatch(InputLogEvent event) { PutLogEventsRequest.Builder requestBuilder = PutLogEventsRequest.builder() .logGroupName(logGroup) .logStreamName(logStream) - .logEvents(events); + .logEvents(List.of(event)); synchronized (this) { if (sequenceToken != null) { @@ -198,78 +201,62 @@ private void executePutLogEvents(PutLogEventsRequest.Builder requestBuilder) { sequenceToken = response.nextSequenceToken(); } - private void ensureLogGroupAndStream() { - try { - client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); - } catch (ResourceAlreadyExistsException ignored) { - LOGGER.debug("Log group {} already exists", logGroup); + private void reapFuturesMap() { + for (Future future : futures.keySet()) { + if (future.isDone()) { + Future.State currFutureState = future.state(); + EventWithRetry currValue = futures.remove(future); + if (currFutureState.equals(Future.State.FAILED) + || currFutureState.equals(Future.State.CANCELLED)) { + if (currValue.retryCount >= 3) { + LOGGER.error("Event retries failed. Event dropped: {}", currValue.inputLogEvent); + } else { + EventWithRetry newValue = + new EventWithRetry(currValue.inputLogEvent, currValue.retryCount + 1); + future = + executorService.submit(() -> sendAndHandleCloudWatchCall(newValue.inputLogEvent)); + futures.put(future, newValue); + } + } + } } + } + private void transformAndSendEvent(HashMap properties) { + properties.put("realm", callContext.getRealmContext().getRealmIdentifier()); + properties.put("principal", securityContext.getUserPrincipal().getName()); + // TODO: Add request ID when it is available + String eventAsJson; try { - client.createLogStream( - CreateLogStreamRequest.builder().logGroupName(logGroup).logStreamName(logStream).build()); - } catch (ResourceAlreadyExistsException ignored) { - LOGGER.debug("Log stream {} already exists", logStream); + eventAsJson = objectMapper.writeValueAsString(properties); + } catch (JsonProcessingException e) { + LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); + return; } - - sequenceToken = getSequenceToken(); + InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp(callContext)); + Future future = executorService.submit(() -> sendAndHandleCloudWatchCall(inputLogEvent)); + futures.put(future, new EventWithRetry(inputLogEvent, 0)); } - private String getSequenceToken() { - DescribeLogStreamsResponse response = - client.describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(logGroup) - .logStreamNamePrefix(logStream) - .build()); - - return response.logStreams().stream() - .filter(s -> logStream.equals(s.logStreamName())) - .map(LogStream::uploadSequenceToken) - .filter(Objects::nonNull) - .findFirst() - .orElse(null); + private long getCurrentTimestamp(CallContext callContext) { + return callContext.getPolarisCallContext().getClock().millis(); } - @PreDestroy - void shutdown() { - running = false; - if (backgroundTask != null) { - try { - backgroundTask.get(10, TimeUnit.SECONDS); - } catch (Exception e) { - LOGGER.error("Error waiting for background logging task to finish: {}", e.getMessage()); - } - } - if (client != null) { - client.close(); - } + private InputLogEvent createLogEvent(String eventAsJson, long timestamp) { + return InputLogEvent.builder().message(eventAsJson).timestamp(timestamp).build(); } - record EventWithTimestamp(String event, long timestamp) {} - - private long getCurrentTimestamp(CallContext callContext) { - return callContext.getPolarisCallContext().getClock().millis(); + @VisibleForTesting + ConcurrentHashMap, EventWithRetry> getFutures() { + return futures; } // Event overrides below - private void queueEvent(PolarisEvent event) { - try { - Map json = objectMapper.convertValue(event, new TypeReference<>() {}); - json.put("realm", callContext.getRealmContext().getRealmIdentifier()); - json.put("event_type", event.getClass().getSimpleName()); - json.put("principal", securityContext.getUserPrincipal().getName()); - // TODO: Add request ID when it is available - queue.add( - new EventWithTimestamp( - objectMapper.writeValueAsString(json), getCurrentTimestamp(callContext))); - } catch (JsonProcessingException e) { - LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); - } - } - @Override - public void onAfterCatalogCreated(AfterCatalogCreatedEvent event) { - queueEvent(event); + public void onAfterTableRefreshed(AfterTableRefreshedEvent event) { + HashMap properties = new HashMap<>(); + properties.put("event_type", event.getClass().getSimpleName()); + properties.put("table_identifier", event.tableIdentifier().toString()); + transformAndSendEvent(properties); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java index f19e4c2ea8..485766bb24 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/PolarisEventListener.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.events; -import org.apache.polaris.core.context.CallContext; - /** * Represents an event listener that can respond to notable moments during Polaris's execution. * Event details are documented under the event objects themselves. @@ -57,7 +55,4 @@ public void onBeforeTaskAttempted(BeforeTaskAttemptedEvent event) {} /** {@link AfterTaskAttemptedEvent} */ public void onAfterTaskAttempted(AfterTaskAttemptedEvent event) {} - - /** {@link AfterCatalogCreatedEvent} */ - public void onAfterCatalogCreated(AfterCatalogCreatedEvent event) {} } diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java index 419a1dc776..5e79a33d45 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java @@ -19,9 +19,12 @@ package org.apache.polaris.service.events; +import static java.lang.Thread.sleep; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; import static org.mockito.Mockito.when; +import jakarta.ws.rs.core.SecurityContext; import java.security.Principal; import java.time.Clock; import java.util.ArrayList; @@ -30,14 +33,10 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; - -import jakarta.ws.rs.core.SecurityContext; +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.FileStorageConfigInfo; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.junit.jupiter.api.AfterEach; @@ -60,7 +59,6 @@ import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsResponse; -import software.amazon.awssdk.services.cloudwatchlogs.model.InputLogEvent; import software.amazon.awssdk.services.cloudwatchlogs.model.OutputLogEvent; class AwsCloudWatchEventListenerTest { @@ -97,7 +95,6 @@ void setUp() { when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); - // Create CloudWatch client pointing to LocalStack cloudWatchLogsClient = CloudWatchLogsClient.builder() @@ -177,21 +174,37 @@ void shouldCreateLogGroupAndStream() { } @Test - void shouldSendEventToCloudWatchSingleEventSubmissions() { + void shouldSendEventToCloudWatch() { listener.start(); - // Create a test catalog entity - String catalog1Name = "test-catalog1"; - String catalog2Name = "test-catalog2"; + // Create test table identifiers + TableIdentifier t1 = TableIdentifier.of("test_ns", "test_table1"); + TableIdentifier t2 = TableIdentifier.of("test_ns", "test_table2"); // Create and send the event - AfterCatalogCreatedEvent event = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); - listener.onAfterCatalogCreated(event); + AfterTableRefreshedEvent event = new AfterTableRefreshedEvent(t1); + listener.onAfterTableRefreshed(event); + + // Verify future and status + List> allActiveFutures = listener.getFutures().keySet().stream().toList(); + assertThat(allActiveFutures).hasSize(1); + Future firstFuture = allActiveFutures.get(0); + + // refactor method + try { + long start = System.currentTimeMillis(); + while (!firstFuture.isDone()) { + if (System.currentTimeMillis() - start > 30 * 1000) { + fail("Future interrupted or did not finish"); + } + sleep(1000); // wait one second before checking again + } + } catch (InterruptedException e) { + fail("Future interrupted or did not finish"); + } - // Wait a bit for the background thread to process - List drainedEvents = new ArrayList<>(); - List transformedEvents = new ArrayList<>(); - listener.drainQueue(drainedEvents, transformedEvents); + assertThat(firstFuture.isDone()).isTrue(); + assertThat(firstFuture.state()).isEqualTo(Future.State.SUCCESS); // Verify the event was sent to CloudWatch GetLogEventsResponse logEvents = @@ -207,21 +220,36 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { .satisfies( logEvent -> { String message = logEvent.message(); - assertThat(message).contains(catalog1Name); assertThat(message).contains(REALM); - assertThat(message).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + assertThat(message).contains(AfterTableRefreshedEvent.class.getSimpleName()); assertThat(message).contains(TEST_USER); + assertThat(message).contains(t1.toString()); }); - assertThat(transformedEvents).isEmpty(); - assertThat(drainedEvents).isEmpty(); - // Redo above procedure to ensure that non-cold-start events can also be submitted - event = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); - listener.onAfterCatalogCreated(event); + event = new AfterTableRefreshedEvent(t2); + listener.onAfterTableRefreshed(event); + + allActiveFutures = listener.getFutures().keySet().stream().toList(); + assertThat(allActiveFutures).hasSize(2); + Future secondFuture = + allActiveFutures.stream().filter((f) -> !f.equals(firstFuture)).findFirst().get(); + + try { + long start = System.currentTimeMillis(); + while (!secondFuture.isDone()) { + if (System.currentTimeMillis() - start > 30 * 1000) { + fail("Future interrupted or did not finish"); + } + sleep(1000); // wait one second before checking again + } + } catch (InterruptedException e) { + fail("Future interrupted or did not finish"); + } - // Wait a bit for the background thread to process - listener.drainQueue(drainedEvents, transformedEvents); + assertThat(secondFuture.isDone()).isTrue(); + assertThat(secondFuture.state()).isEqualTo(Future.State.SUCCESS); + assertThat(listener.getFutures().get(firstFuture)).isEqualTo(null); // first future got purged // Verify the event was sent to CloudWatch logEvents = @@ -235,56 +263,9 @@ void shouldSendEventToCloudWatchSingleEventSubmissions() { List sortedEvents = new ArrayList<>(logEvents.events()); sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); String secondMsg = sortedEvents.get(1).message(); - assertThat(secondMsg).contains(catalog2Name); assertThat(secondMsg).contains(REALM); - assertThat(secondMsg).contains(AfterCatalogCreatedEvent.class.getSimpleName()); + assertThat(secondMsg).contains(AfterTableRefreshedEvent.class.getSimpleName()); assertThat(secondMsg).contains(TEST_USER); - - assertThat(transformedEvents).isEmpty(); - assertThat(drainedEvents).isEmpty(); - } - - @Test - void shouldSendEventToCloudWatchBatchEventSubmissions() { - listener.start(); - String catalog1Name = "test-catalog1"; - String catalog2Name = "test-catalog2"; - - AfterCatalogCreatedEvent event1 = new AfterCatalogCreatedEvent(getTestCatalog(catalog1Name)); - AfterCatalogCreatedEvent event2 = new AfterCatalogCreatedEvent(getTestCatalog(catalog2Name)); - - listener.onAfterCatalogCreated(event1); - listener.onAfterCatalogCreated(event2); - List drainedEvents = new ArrayList<>(); - List transformedEvents = new ArrayList<>(); - listener.drainQueue(drainedEvents, transformedEvents); - - // Verify the events were sent to CloudWatch - GetLogEventsResponse logEvents = - cloudWatchLogsClient.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()); - - assertThat(logEvents.events()).hasSize(2); - List sortedEvents = new ArrayList<>(logEvents.events()); - sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); - assertThat(sortedEvents.get(0).message()).contains(catalog1Name); - assertThat(sortedEvents.get(1).message()).contains(catalog2Name); - - assertThat(transformedEvents).isEmpty(); - assertThat(drainedEvents).isEmpty(); - } - - private Catalog getTestCatalog(String catalogName) { - return PolarisCatalog.builder() - .setName(catalogName) - .setType(Catalog.TypeEnum.INTERNAL) - .setStorageConfigInfo( - FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://")) - .build()) - .build(); + assertThat(secondMsg).contains(t2.toString()); } } diff --git a/site/content/in-dev/unreleased/configuration.md b/site/content/in-dev/unreleased/configuration.md index fec8940d6b..750d808219 100644 --- a/site/content/in-dev/unreleased/configuration.md +++ b/site/content/in-dev/unreleased/configuration.md @@ -118,6 +118,10 @@ read-only mode, as Polaris only reads the configuration file once, at startup. | `polaris.tasks.max-concurrent-tasks` | `100` | Define the max number of concurrent tasks. | | `polaris.tasks.max-queued-tasks` | `1000` | Define the max number of tasks in queue. | | `polaris.config.rollback.compaction.on-conflicts.enabled` | `false` | When set to true Polaris will apply the deconfliction by rollbacking those REPLACE operations snapshots which have the property of `polaris.internal.rollback.compaction.on-conflict` in their snapshot summary set to `rollback`, to resolve conflicts at the server end. | +| `polaris.event-listener.type` | `no-op` | Define the Polaris event listener type. Supported values are `no-op`, `aws-cloudwatch`. | +| `polaris.event-listener.aws-cloudwatch.log-group` | | Define the AWS CloudWatch log group name for the event listener. | +| `polaris.event-listener.aws-cloudwatch.log-stream` | | Define the AWS CloudWatch log stream name for the event listener. | +| `polaris.event-listener.aws-cloudwatch.region` | | Define the AWS region for the CloudWatch event listener. | There are non Polaris configuration properties that can be useful: From f3f62a09159f718444c7b453d4b869867b852b30 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Sun, 20 Jul 2025 21:45:46 -0700 Subject: [PATCH 14/39] resolve comments from @eric-maynard and @snazy --- ...rkusPolarisEventListenerConfiguration.java | 4 + service/common/build.gradle.kts | 1 + .../events/AwsCloudWatchEventListener.java | 23 +- .../AwsCloudWatchEventListenerTest.java | 352 +++++++++++------- .../events/Dockerfile-localstack-version | 23 ++ 5 files changed, 257 insertions(+), 146 deletions(-) create mode 100644 service/common/src/test/resources/org/apache/polaris/service/events/Dockerfile-localstack-version diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java index 05bb9fb1b2..e740ef1079 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java @@ -20,6 +20,7 @@ import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; import io.smallrye.config.WithName; import java.util.Optional; import org.apache.polaris.service.events.EventListenerConfiguration; @@ -35,14 +36,17 @@ public interface QuarkusPolarisEventListenerConfiguration extends EventListenerC String type(); @WithName("aws-cloudwatch.log-group") + @WithDefault("polaris-cloudwatch-default-group") @Override Optional awsCloudwatchlogGroup(); @WithName("aws-cloudwatch.log-stream") + @WithDefault("polaris-cloudwatch-default-stream") @Override Optional awsCloudwatchlogStream(); @WithName("aws-cloudwatch.region") + @WithDefault("us-east-1") @Override Optional awsCloudwatchRegion(); } diff --git a/service/common/build.gradle.kts b/service/common/build.gradle.kts index ab1dbc797b..060c15faae 100644 --- a/service/common/build.gradle.kts +++ b/service/common/build.gradle.kts @@ -97,6 +97,7 @@ dependencies { testImplementation(libs.mockito.core) testImplementation(libs.localstack) testImplementation("org.testcontainers:testcontainers") + testImplementation(project(":polaris-container-spec-helper")) testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation(libs.logback.classic) diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java index ecd0c93697..381e50b823 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java @@ -61,9 +61,6 @@ public class AwsCloudWatchEventListener extends PolarisEventListener { private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListener.class); private final ObjectMapper objectMapper = new ObjectMapper(); private final ConcurrentHashMap, EventWithRetry> futures = new ConcurrentHashMap<>(); - private static final String DEFAULT_LOG_STREAM_NAME = "polaris-cloudwatch-default-stream"; - private static final String DEFAULT_LOG_GROUP_NAME = "polaris-cloudwatch-default-group"; - private static final String DEFAULT_REGION = "us-east-1"; record EventWithRetry(InputLogEvent inputLogEvent, int retryCount) {} @@ -85,9 +82,23 @@ public AwsCloudWatchEventListener( EventListenerConfiguration config, ExecutorService executorService) { this.executorService = executorService; - this.logStream = config.awsCloudwatchlogStream().orElse(DEFAULT_LOG_STREAM_NAME); - this.logGroup = config.awsCloudwatchlogGroup().orElse(DEFAULT_LOG_GROUP_NAME); - this.region = Region.of(config.awsCloudwatchRegion().orElse(DEFAULT_REGION)); + this.logStream = + config + .awsCloudwatchlogStream() + .orElseThrow( + () -> new IllegalArgumentException("AWS CloudWatch log stream must be configured")); + this.logGroup = + config + .awsCloudwatchlogGroup() + .orElseThrow( + () -> new IllegalArgumentException("AWS CloudWatch log group must be configured")); + this.region = + Region.of( + config + .awsCloudwatchRegion() + .orElseThrow( + () -> + new IllegalArgumentException("AWS CloudWatch region must be configured"))); } @PostConstruct diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java index 5e79a33d45..f2469a41b2 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java @@ -19,57 +19,63 @@ package org.apache.polaris.service.events; -import static java.lang.Thread.sleep; +import static org.apache.polaris.containerspec.ContainerSpecHelper.containerSpecHelper; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import jakarta.ws.rs.core.SecurityContext; import java.security.Principal; import java.time.Clock; -import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.localstack.LocalStackContainer; -import org.testcontainers.utility.DockerImageName; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; +import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsResponse; -import software.amazon.awssdk.services.cloudwatchlogs.model.OutputLogEvent; +import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsResponse; class AwsCloudWatchEventListenerTest { private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListenerTest.class); - private static final DockerImageName LOCALSTACK_IMAGE = - DockerImageName.parse("localstack/localstack:3.4"); - private static final LocalStackContainer localStack = - new LocalStackContainer(LOCALSTACK_IMAGE) + new LocalStackContainer( + containerSpecHelper("localstack", AwsCloudWatchEventListenerTest.class) + .dockerImageName(null)) .withServices(LocalStackContainer.Service.CLOUDWATCHLOGS); private static final String LOG_GROUP = "test-log-group"; @@ -80,13 +86,20 @@ class AwsCloudWatchEventListenerTest { @Mock private EventListenerConfiguration config; private ExecutorService executorService; - private AwsCloudWatchEventListener listener; - private CloudWatchLogsClient cloudWatchLogsClient; private AutoCloseable mockitoContext; + enum TestMode { + LOCALSTACK, + MOCKED + } + + // Test data provider + static Stream testModeProvider() { + return Stream.of(Arguments.of(TestMode.LOCALSTACK), Arguments.of(TestMode.MOCKED)); + } + @BeforeEach void setUp() { - localStack.start(); mockitoContext = MockitoAnnotations.openMocks(this); executorService = Executors.newSingleThreadExecutor(); @@ -94,10 +107,31 @@ void setUp() { when(config.awsCloudwatchlogGroup()).thenReturn(Optional.of(LOG_GROUP)); when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); + } + + @AfterEach + void tearDown() throws Exception { + if (mockitoContext != null) { + mockitoContext.close(); + } + if (executorService != null) { + executorService.shutdownNow(); + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + LOGGER.warn("ExecutorService did not terminate in time"); + } + } + if (localStack.isRunning()) { + localStack.stop(); + } + } - // Create CloudWatch client pointing to LocalStack - cloudWatchLogsClient = - CloudWatchLogsClient.builder() + private CloudWatchLogsClient createCloudWatchClient(TestMode mode) { + switch (mode) { + case LOCALSTACK: + if (!localStack.isRunning()) { + localStack.start(); + } + return CloudWatchLogsClient.builder() .endpointOverride(localStack.getEndpoint()) .credentialsProvider( StaticCredentialsProvider.create( @@ -105,15 +139,46 @@ void setUp() { localStack.getAccessKey(), localStack.getSecretKey()))) .region(Region.of(localStack.getRegion())) .build(); + case MOCKED: + CloudWatchLogsClient mockClient = Mockito.mock(CloudWatchLogsClient.class); + + // Mock the responses for log group and stream creation + when(mockClient.createLogGroup(any(CreateLogGroupRequest.class))).thenReturn(null); + when(mockClient.createLogStream(any(CreateLogStreamRequest.class))).thenReturn(null); + + // Mock the describe log streams response for getting sequence token + DescribeLogStreamsResponse mockStreamsResponse = + DescribeLogStreamsResponse.builder() + .logStreams( + software.amazon.awssdk.services.cloudwatchlogs.model.LogStream.builder() + .logStreamName(LOG_STREAM) + .uploadSequenceToken(null) + .build()) + .build(); + when(mockClient.describeLogStreams(any(DescribeLogStreamsRequest.class))) + .thenReturn(mockStreamsResponse); + + // Mock the putLogEvents response + PutLogEventsResponse mockPutResponse = + PutLogEventsResponse.builder().nextSequenceToken("mock-sequence-token-123").build(); + when(mockClient.putLogEvents(any(PutLogEventsRequest.class))).thenReturn(mockPutResponse); + + return mockClient; + default: + throw new IllegalArgumentException("Unknown test mode: " + mode); + } + } - listener = + private AwsCloudWatchEventListener createListener(CloudWatchLogsClient client) { + AwsCloudWatchEventListener listener = new AwsCloudWatchEventListener(config, executorService) { @Override protected CloudWatchLogsClient createCloudWatchClient() { - return cloudWatchLogsClient; + return client; } }; + // Set up call context and security context CallContext callContext = Mockito.mock(CallContext.class); PolarisCallContext polarisCallContext = Mockito.mock(PolarisCallContext.class); RealmContext realmContext = Mockito.mock(RealmContext.class); @@ -127,145 +192,152 @@ protected CloudWatchLogsClient createCloudWatchClient() { when(principal.getName()).thenReturn(TEST_USER); listener.callContext = callContext; listener.securityContext = securityContext; + + return listener; } - @AfterEach - void tearDown() throws Exception { - if (mockitoContext != null) { - mockitoContext.close(); - } - if (executorService != null) { - executorService.shutdownNow(); - if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { - LOGGER.warn("ExecutorService did not terminate in time"); + private void waitForFuture(Future future, long timeoutMs) { + try { + long start = System.currentTimeMillis(); + while (!future.isDone()) { + if (System.currentTimeMillis() - start > timeoutMs) { + fail("Future did not complete in time"); + } + Thread.sleep(100); } + } catch (InterruptedException e) { + fail("Future was interrupted"); } - if (cloudWatchLogsClient != null) { - cloudWatchLogsClient.close(); - } - localStack.stop(); } - @Test - void shouldCreateLogGroupAndStream() { + @ParameterizedTest + @MethodSource("testModeProvider") + void shouldCreateLogGroupAndStream(TestMode mode) { + CloudWatchLogsClient client = createCloudWatchClient(mode); + AwsCloudWatchEventListener listener = createListener(client); + // Start the listener which should create the log group and stream listener.start(); - // Verify log group exists - DescribeLogGroupsResponse groups = - cloudWatchLogsClient.describeLogGroups( - DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()); - assertThat(groups.logGroups()) - .hasSize(1) - .first() - .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); - - // Verify log stream exists - DescribeLogStreamsResponse streams = - cloudWatchLogsClient.describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamNamePrefix(LOG_STREAM) - .build()); - assertThat(streams.logStreams()) - .hasSize(1) - .first() - .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + if (mode == TestMode.LOCALSTACK) { + // Verify log group exists + DescribeLogGroupsResponse groups = + client.describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()); + assertThat(groups.logGroups()) + .hasSize(1) + .first() + .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); + + // Verify log stream exists + DescribeLogStreamsResponse streams = + client.describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamNamePrefix(LOG_STREAM) + .build()); + assertThat(streams.logStreams()) + .hasSize(1) + .first() + .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + } else { + // Verify method calls for mocked client + verify(client, times(1)) + .createLogGroup( + argThat((CreateLogGroupRequest request) -> request.logGroupName().equals(LOG_GROUP))); + verify(client, times(1)) + .createLogStream( + argThat( + (CreateLogStreamRequest request) -> + request.logGroupName().equals(LOG_GROUP) + && request.logStreamName().equals(LOG_STREAM))); + verify(client, times(1)) + .describeLogStreams( + argThat( + (DescribeLogStreamsRequest request) -> + request.logGroupName().equals(LOG_GROUP) + && request.logStreamNamePrefix().equals(LOG_STREAM))); + } + + // Clean up + listener.shutdown(); + if (client != null && mode == TestMode.LOCALSTACK) { + client.close(); + } } - @Test - void shouldSendEventToCloudWatch() { - listener.start(); + @ParameterizedTest + @MethodSource("testModeProvider") + void shouldSendEventToCloudWatch(TestMode mode) { + CloudWatchLogsClient client = createCloudWatchClient(mode); + AwsCloudWatchEventListener listener = createListener(client); - // Create test table identifiers - TableIdentifier t1 = TableIdentifier.of("test_ns", "test_table1"); - TableIdentifier t2 = TableIdentifier.of("test_ns", "test_table2"); + // Start the listener + listener.start(); - // Create and send the event - AfterTableRefreshedEvent event = new AfterTableRefreshedEvent(t1); + // Create and send a test event + TableIdentifier testTable = TableIdentifier.of("test_namespace", "test_table"); + AfterTableRefreshedEvent event = new AfterTableRefreshedEvent(testTable); listener.onAfterTableRefreshed(event); - // Verify future and status - List> allActiveFutures = listener.getFutures().keySet().stream().toList(); - assertThat(allActiveFutures).hasSize(1); - Future firstFuture = allActiveFutures.get(0); - - // refactor method - try { - long start = System.currentTimeMillis(); - while (!firstFuture.isDone()) { - if (System.currentTimeMillis() - start > 30 * 1000) { - fail("Future interrupted or did not finish"); - } - sleep(1000); // wait one second before checking again - } - } catch (InterruptedException e) { - fail("Future interrupted or did not finish"); + // Wait for the future to complete + List> activeFutures = listener.getFutures().keySet().stream().toList(); + assertThat(activeFutures).hasSize(1); + Future eventFuture = activeFutures.getFirst(); + + long timeout = mode == TestMode.MOCKED ? 5000L : 30000L; // Shorter timeout for mocked tests + waitForFuture(eventFuture, timeout); + + // Verify the future completed successfully + assertThat(eventFuture.isDone()).isTrue(); + assertThat(eventFuture.state()).isEqualTo(Future.State.SUCCESS); + + if (mode == TestMode.LOCALSTACK) { + // Verify the event was sent to CloudWatch by reading the actual logs + GetLogEventsResponse logEvents = + client.getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()); + + assertThat(logEvents.events()) + .hasSize(1) + .first() + .satisfies( + logEvent -> { + String message = logEvent.message(); + assertThat(message).contains(REALM); + assertThat(message).contains(AfterTableRefreshedEvent.class.getSimpleName()); + assertThat(message).contains(TEST_USER); + assertThat(message).contains(testTable.toString()); + }); + } else { + // Verify that putLogEvents was called with the expected content + verify(client, times(1)) + .putLogEvents( + argThat( + (PutLogEventsRequest request) -> { + // Verify basic request structure + assertThat(request.logGroupName()).isEqualTo(LOG_GROUP); + assertThat(request.logStreamName()).isEqualTo(LOG_STREAM); + assertThat(request.logEvents()).hasSize(1); + + // Verify the log event content + String logMessage = request.logEvents().getFirst().message(); + assertThat(logMessage).contains(REALM); + assertThat(logMessage).contains(TEST_USER); + assertThat(logMessage).contains("AfterTableRefreshedEvent"); + assertThat(logMessage).contains(testTable.toString()); + + return true; + })); } - assertThat(firstFuture.isDone()).isTrue(); - assertThat(firstFuture.state()).isEqualTo(Future.State.SUCCESS); - - // Verify the event was sent to CloudWatch - GetLogEventsResponse logEvents = - cloudWatchLogsClient.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()); - - assertThat(logEvents.events()) - .hasSize(1) - .first() - .satisfies( - logEvent -> { - String message = logEvent.message(); - assertThat(message).contains(REALM); - assertThat(message).contains(AfterTableRefreshedEvent.class.getSimpleName()); - assertThat(message).contains(TEST_USER); - assertThat(message).contains(t1.toString()); - }); - - // Redo above procedure to ensure that non-cold-start events can also be submitted - event = new AfterTableRefreshedEvent(t2); - listener.onAfterTableRefreshed(event); - - allActiveFutures = listener.getFutures().keySet().stream().toList(); - assertThat(allActiveFutures).hasSize(2); - Future secondFuture = - allActiveFutures.stream().filter((f) -> !f.equals(firstFuture)).findFirst().get(); - - try { - long start = System.currentTimeMillis(); - while (!secondFuture.isDone()) { - if (System.currentTimeMillis() - start > 30 * 1000) { - fail("Future interrupted or did not finish"); - } - sleep(1000); // wait one second before checking again - } - } catch (InterruptedException e) { - fail("Future interrupted or did not finish"); + // Clean up + listener.shutdown(); + if (client != null && mode == TestMode.LOCALSTACK) { + client.close(); } - - assertThat(secondFuture.isDone()).isTrue(); - assertThat(secondFuture.state()).isEqualTo(Future.State.SUCCESS); - assertThat(listener.getFutures().get(firstFuture)).isEqualTo(null); // first future got purged - - // Verify the event was sent to CloudWatch - logEvents = - cloudWatchLogsClient.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()); - - assertThat(logEvents.events()).hasSize(2); - List sortedEvents = new ArrayList<>(logEvents.events()); - sortedEvents.sort(Comparator.comparingLong(OutputLogEvent::timestamp)); - String secondMsg = sortedEvents.get(1).message(); - assertThat(secondMsg).contains(REALM); - assertThat(secondMsg).contains(AfterTableRefreshedEvent.class.getSimpleName()); - assertThat(secondMsg).contains(TEST_USER); - assertThat(secondMsg).contains(t2.toString()); } } diff --git a/service/common/src/test/resources/org/apache/polaris/service/events/Dockerfile-localstack-version b/service/common/src/test/resources/org/apache/polaris/service/events/Dockerfile-localstack-version new file mode 100644 index 0000000000..2e17ab71e8 --- /dev/null +++ b/service/common/src/test/resources/org/apache/polaris/service/events/Dockerfile-localstack-version @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM localstack/localstack:3.4 + From d21dabc1c10eb9036f3edf181ed41f4a2decb84e Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 21 Jul 2025 00:16:01 -0700 Subject: [PATCH 15/39] refactor into separate package --- .../quarkus/events/AwsCloudwatchConfig.java | 28 ----------- ...rkusPolarisEventListenerConfiguration.java | 21 +------- .../QuarkusAwsCloudWatchConfiguration.java | 48 +++++++++++++++++++ .../events/AfterCatalogCreatedEvent.java | 29 ----------- .../AwsCloudWatchConfiguration.java} | 5 +- .../AwsCloudWatchEventListener.java | 7 ++- .../AwsCloudWatchEventListenerTest.java | 5 +- .../cloudwatch}/Dockerfile-localstack-version | 0 8 files changed, 60 insertions(+), 83 deletions(-) delete mode 100644 runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java create mode 100644 runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java delete mode 100644 service/common/src/main/java/org/apache/polaris/service/events/AfterCatalogCreatedEvent.java rename service/common/src/main/java/org/apache/polaris/service/events/{EventListenerConfiguration.java => aws/cloudwatch/AwsCloudWatchConfiguration.java} (84%) rename service/common/src/main/java/org/apache/polaris/service/events/{ => aws/cloudwatch}/AwsCloudWatchEventListener.java (96%) rename service/common/src/test/java/org/apache/polaris/service/events/{ => aws/cloudwatch}/AwsCloudWatchEventListenerTest.java (98%) rename service/common/src/test/resources/org/apache/polaris/service/events/{ => aws/cloudwatch}/Dockerfile-localstack-version (100%) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java deleted file mode 100644 index d4536f3437..0000000000 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/AwsCloudwatchConfig.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.polaris.service.quarkus.events; - -public interface AwsCloudwatchConfig { - String logGroup(); - - String logStream(); - - String region(); -} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java index e740ef1079..3444992e8a 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/QuarkusPolarisEventListenerConfiguration.java @@ -20,33 +20,14 @@ import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; -import io.smallrye.config.WithDefault; -import io.smallrye.config.WithName; -import java.util.Optional; -import org.apache.polaris.service.events.EventListenerConfiguration; import org.apache.polaris.service.events.PolarisEventListener; @StaticInitSafe @ConfigMapping(prefix = "polaris.event-listener") -public interface QuarkusPolarisEventListenerConfiguration extends EventListenerConfiguration { +public interface QuarkusPolarisEventListenerConfiguration { /** * The type of the event listener to use. Must be a registered {@link PolarisEventListener} * identifier. */ String type(); - - @WithName("aws-cloudwatch.log-group") - @WithDefault("polaris-cloudwatch-default-group") - @Override - Optional awsCloudwatchlogGroup(); - - @WithName("aws-cloudwatch.log-stream") - @WithDefault("polaris-cloudwatch-default-stream") - @Override - Optional awsCloudwatchlogStream(); - - @WithName("aws-cloudwatch.region") - @WithDefault("us-east-1") - @Override - Optional awsCloudwatchRegion(); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java new file mode 100644 index 0000000000..c6998eacba --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.events.aws.cloudwatch; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; +import io.smallrye.config.WithName; +import jakarta.enterprise.context.ApplicationScoped; +import java.util.Optional; +import org.apache.polaris.service.events.aws.cloudwatch.AwsCloudWatchConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.event-listener.aws-cloudwatch") +@ApplicationScoped +public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfiguration { + + @WithName("log-group") + @WithDefault("polaris-cloudwatch-default-group") + @Override + Optional awsCloudwatchlogGroup(); + + @WithName("log-stream") + @WithDefault("polaris-cloudwatch-default-stream") + @Override + Optional awsCloudwatchlogStream(); + + @WithName("region") + @WithDefault("us-east-1") + @Override + Optional awsCloudwatchRegion(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AfterCatalogCreatedEvent.java b/service/common/src/main/java/org/apache/polaris/service/events/AfterCatalogCreatedEvent.java deleted file mode 100644 index 300ae1a4e2..0000000000 --- a/service/common/src/main/java/org/apache/polaris/service/events/AfterCatalogCreatedEvent.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.events; - -import org.apache.polaris.core.admin.model.Catalog; - -/** - * Emitted after Polaris creates a catalog (internal or external). This is not emitted if there's an - * exception while created. - * - * @param catalog The catalog that was created - */ -public record AfterCatalogCreatedEvent(Catalog catalog) implements PolarisEvent {} diff --git a/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchConfiguration.java similarity index 84% rename from service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java rename to service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchConfiguration.java index 701e1a3754..e4cbbe2ec2 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/EventListenerConfiguration.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchConfiguration.java @@ -17,11 +17,12 @@ * under the License. */ -package org.apache.polaris.service.events; +package org.apache.polaris.service.events.aws.cloudwatch; import java.util.Optional; -public interface EventListenerConfiguration { +/** Configuration interface for AWS CloudWatch event listener settings. */ +public interface AwsCloudWatchConfiguration { Optional awsCloudwatchlogGroup(); Optional awsCloudwatchlogStream(); diff --git a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java similarity index 96% rename from service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java rename to service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java index 381e50b823..4e3bd6c7f7 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.service.events; +package org.apache.polaris.service.events.aws.cloudwatch; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -36,6 +36,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.service.events.AfterCatalogCreatedEvent; +import org.apache.polaris.service.events.AfterTableRefreshedEvent; +import org.apache.polaris.service.events.PolarisEventListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.exception.SdkClientException; @@ -79,7 +82,7 @@ record EventWithRetry(InputLogEvent inputLogEvent, int retryCount) {} @Inject public AwsCloudWatchEventListener( - EventListenerConfiguration config, ExecutorService executorService) { + AwsCloudWatchConfiguration config, ExecutorService executorService) { this.executorService = executorService; this.logStream = diff --git a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListenerTest.java similarity index 98% rename from service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java rename to service/common/src/test/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index f2469a41b2..cb9fdd80e3 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.service.events; +package org.apache.polaris.service.events.aws.cloudwatch; import static org.apache.polaris.containerspec.ContainerSpecHelper.containerSpecHelper; import static org.assertj.core.api.Assertions.assertThat; @@ -42,6 +42,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.service.events.AfterTableRefreshedEvent; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; @@ -83,7 +84,7 @@ class AwsCloudWatchEventListenerTest { private static final String REALM = "test-realm"; private static final String TEST_USER = "test-user"; - @Mock private EventListenerConfiguration config; + @Mock private AwsCloudWatchConfiguration config; private ExecutorService executorService; private AutoCloseable mockitoContext; diff --git a/service/common/src/test/resources/org/apache/polaris/service/events/Dockerfile-localstack-version b/service/common/src/test/resources/org/apache/polaris/service/events/aws/cloudwatch/Dockerfile-localstack-version similarity index 100% rename from service/common/src/test/resources/org/apache/polaris/service/events/Dockerfile-localstack-version rename to service/common/src/test/resources/org/apache/polaris/service/events/aws/cloudwatch/Dockerfile-localstack-version From 905451137c8100e6bb9c788ecb1b5abf1fa0b041 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 21 Jul 2025 00:27:57 -0700 Subject: [PATCH 16/39] typo --- .../events/aws/cloudwatch/AwsCloudWatchEventListener.java | 1 - 1 file changed, 1 deletion(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java index 4e3bd6c7f7..c660975f51 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -36,7 +36,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.events.AfterCatalogCreatedEvent; import org.apache.polaris.service.events.AfterTableRefreshedEvent; import org.apache.polaris.service.events.PolarisEventListener; import org.slf4j.Logger; From 828760ac4d0d4aa82107e2d4beb1d4f26a94aa1c Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 22 Jul 2025 03:01:48 -0700 Subject: [PATCH 17/39] revising comments from @eric-maynard --- .../QuarkusAwsCloudWatchConfiguration.java | 9 ++- .../jsonEventListener/JsonEventListener.java | 38 +++++++++++ .../AwsCloudWatchConfiguration.java | 4 +- .../AwsCloudWatchEventListener.java | 27 ++++---- .../AwsCloudWatchEventListenerTest.java | 66 ++++++++++++++++++- .../cloudwatch/Dockerfile-localstack-version | 0 6 files changed, 126 insertions(+), 18 deletions(-) rename runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/{ => jsonEventListener}/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java (85%) create mode 100644 service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java rename service/common/src/main/java/org/apache/polaris/service/events/{ => jsonEventListener}/aws/cloudwatch/AwsCloudWatchConfiguration.java (91%) rename service/common/src/main/java/org/apache/polaris/service/events/{ => jsonEventListener}/aws/cloudwatch/AwsCloudWatchEventListener.java (93%) rename service/common/src/test/java/org/apache/polaris/service/events/{ => jsonEventListener}/aws/cloudwatch/AwsCloudWatchEventListenerTest.java (83%) rename service/common/src/test/resources/org/apache/polaris/service/events/{ => jsonEventListener}/aws/cloudwatch/Dockerfile-localstack-version (100%) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java similarity index 85% rename from runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java rename to runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java index c6998eacba..d7d56d739d 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.quarkus.events.aws.cloudwatch; +package org.apache.polaris.service.quarkus.events.jsonEventListener.aws.cloudwatch; import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; @@ -24,7 +24,7 @@ import io.smallrye.config.WithName; import jakarta.enterprise.context.ApplicationScoped; import java.util.Optional; -import org.apache.polaris.service.events.aws.cloudwatch.AwsCloudWatchConfiguration; +import org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch.AwsCloudWatchConfiguration; @StaticInitSafe @ConfigMapping(prefix = "polaris.event-listener.aws-cloudwatch") @@ -45,4 +45,9 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur @WithDefault("us-east-1") @Override Optional awsCloudwatchRegion(); + + @WithName("synchronous-mode") + @WithDefault("false") + @Override + String synchronousMode(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java new file mode 100644 index 0000000000..9c0a56e099 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.events.jsonEventListener; + +import org.apache.polaris.service.events.AfterTableRefreshedEvent; +import org.apache.polaris.service.events.PolarisEventListener; + +import java.util.HashMap; + +public abstract class JsonEventListener extends PolarisEventListener { + protected abstract void transformAndSendEvent(HashMap properties); + + @Override + public void onAfterTableRefreshed(AfterTableRefreshedEvent event) { + HashMap properties = new HashMap<>(); + properties.put("event_type", event.getClass().getSimpleName()); + properties.put("table_identifier", event.tableIdentifier().toString()); + transformAndSendEvent(properties); + } + +} diff --git a/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java similarity index 91% rename from service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchConfiguration.java rename to service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java index e4cbbe2ec2..5e6234b202 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchConfiguration.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.service.events.aws.cloudwatch; +package org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch; import java.util.Optional; @@ -28,4 +28,6 @@ public interface AwsCloudWatchConfiguration { Optional awsCloudwatchlogStream(); Optional awsCloudwatchRegion(); + + String synchronousMode(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java similarity index 93% rename from service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java rename to service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index c660975f51..e94356f6b2 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.service.events.aws.cloudwatch; +package org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -38,6 +38,7 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.service.events.AfterTableRefreshedEvent; import org.apache.polaris.service.events.PolarisEventListener; +import org.apache.polaris.service.events.jsonEventListener.JsonEventListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.exception.SdkClientException; @@ -59,7 +60,7 @@ @ApplicationScoped @Identifier("aws-cloudwatch") -public class AwsCloudWatchEventListener extends PolarisEventListener { +public class AwsCloudWatchEventListener extends JsonEventListener { private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListener.class); private final ObjectMapper objectMapper = new ObjectMapper(); private final ConcurrentHashMap, EventWithRetry> futures = new ConcurrentHashMap<>(); @@ -74,6 +75,7 @@ record EventWithRetry(InputLogEvent inputLogEvent, int retryCount) {} private final String logGroup; private final String logStream; private final Region region; + private final boolean synchronousMode; @Inject CallContext callContext; @@ -101,6 +103,7 @@ public AwsCloudWatchEventListener( .orElseThrow( () -> new IllegalArgumentException("AWS CloudWatch region must be configured"))); + this.synchronousMode = Boolean.parseBoolean(config.synchronousMode()); } @PostConstruct @@ -235,7 +238,8 @@ private void reapFuturesMap() { } } - private void transformAndSendEvent(HashMap properties) { + @Override + protected void transformAndSendEvent(HashMap properties) { properties.put("realm", callContext.getRealmContext().getRealmIdentifier()); properties.put("principal", securityContext.getUserPrincipal().getName()); // TODO: Add request ID when it is available @@ -247,8 +251,12 @@ private void transformAndSendEvent(HashMap properties) { return; } InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp(callContext)); - Future future = executorService.submit(() -> sendAndHandleCloudWatchCall(inputLogEvent)); - futures.put(future, new EventWithRetry(inputLogEvent, 0)); + if (!synchronousMode) { + Future future = executorService.submit(() -> sendAndHandleCloudWatchCall(inputLogEvent)); + futures.put(future, new EventWithRetry(inputLogEvent, 0)); + } else { + sendAndHandleCloudWatchCall(inputLogEvent); + } } private long getCurrentTimestamp(CallContext callContext) { @@ -263,13 +271,4 @@ private InputLogEvent createLogEvent(String eventAsJson, long timestamp) { ConcurrentHashMap, EventWithRetry> getFutures() { return futures; } - - // Event overrides below - @Override - public void onAfterTableRefreshed(AfterTableRefreshedEvent event) { - HashMap properties = new HashMap<>(); - properties.put("event_type", event.getClass().getSimpleName()); - properties.put("table_identifier", event.tableIdentifier().toString()); - transformAndSendEvent(properties); - } } diff --git a/service/common/src/test/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java similarity index 83% rename from service/common/src/test/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListenerTest.java rename to service/common/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index cb9fdd80e3..ef6e9467dc 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.service.events.aws.cloudwatch; +package org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch; import static org.apache.polaris.containerspec.ContainerSpecHelper.containerSpecHelper; import static org.assertj.core.api.Assertions.assertThat; @@ -108,6 +108,7 @@ void setUp() { when(config.awsCloudwatchlogGroup()).thenReturn(Optional.of(LOG_GROUP)); when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); + when(config.synchronousMode()).thenReturn("false"); // Default to async mode } @AfterEach @@ -341,4 +342,67 @@ void shouldSendEventToCloudWatch(TestMode mode) { client.close(); } } + + @ParameterizedTest + @MethodSource("testModeProvider") + void handleSynchronousModeCorrectly(TestMode mode) { + CloudWatchLogsClient client = createCloudWatchClient(mode); + + // Test synchronous mode + when(config.synchronousMode()).thenReturn("true"); + AwsCloudWatchEventListener syncListener = createListener(client); + syncListener.start(); + + // Create and send a test event synchronously + TableIdentifier syncTestTable = TableIdentifier.of("test_namespace", "test_table_sync"); + AfterTableRefreshedEvent syncEvent = new AfterTableRefreshedEvent(syncTestTable); + syncListener.onAfterTableRefreshed(syncEvent); + assertThat(syncListener.getFutures()).isEmpty(); // No futures should be created in sync mode + + if (mode == TestMode.LOCALSTACK) { + // Verify both events were sent to CloudWatch + GetLogEventsResponse logEvents = + client.getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()); + + assertThat(logEvents.events()).hasSize(1); + + // Verify sync event + assertThat(logEvents.events()) + .anySatisfy(logEvent -> { + String message = logEvent.message(); + assertThat(message).contains("test_table_sync"); + assertThat(message).contains("AfterTableRefreshedEvent"); + }); + } else { + // Verify that putLogEvents was called with the expected content + verify(client, times(1)) + .putLogEvents( + argThat( + (PutLogEventsRequest request) -> { + // Verify basic request structure + assertThat(request.logGroupName()).isEqualTo(LOG_GROUP); + assertThat(request.logStreamName()).isEqualTo(LOG_STREAM); + assertThat(request.logEvents()).hasSize(1); + + // Verify the log event content + String logMessage = request.logEvents().getFirst().message(); + assertThat(logMessage).contains(REALM); + assertThat(logMessage).contains(TEST_USER); + assertThat(logMessage).contains("AfterTableRefreshedEvent"); + assertThat(logMessage).contains("test_table_sync"); + + return true; + })); + } + + // Clean up + syncListener.shutdown(); + if (client != null && mode == TestMode.LOCALSTACK) { + client.close(); + } + } } diff --git a/service/common/src/test/resources/org/apache/polaris/service/events/aws/cloudwatch/Dockerfile-localstack-version b/service/common/src/test/resources/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/Dockerfile-localstack-version similarity index 100% rename from service/common/src/test/resources/org/apache/polaris/service/events/aws/cloudwatch/Dockerfile-localstack-version rename to service/common/src/test/resources/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/Dockerfile-localstack-version From 9d47684c95032521ece6870f6a76ec00f07017f3 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 22 Jul 2025 03:18:19 -0700 Subject: [PATCH 18/39] spotlessapply --- .../jsonEventListener/JsonEventListener.java | 20 +++++++++---------- .../AwsCloudWatchEventListener.java | 2 -- .../AwsCloudWatchEventListenerTest.java | 11 +++++----- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java index 9c0a56e099..5af945e92a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java @@ -19,20 +19,18 @@ package org.apache.polaris.service.events.jsonEventListener; +import java.util.HashMap; import org.apache.polaris.service.events.AfterTableRefreshedEvent; import org.apache.polaris.service.events.PolarisEventListener; -import java.util.HashMap; - public abstract class JsonEventListener extends PolarisEventListener { - protected abstract void transformAndSendEvent(HashMap properties); - - @Override - public void onAfterTableRefreshed(AfterTableRefreshedEvent event) { - HashMap properties = new HashMap<>(); - properties.put("event_type", event.getClass().getSimpleName()); - properties.put("table_identifier", event.tableIdentifier().toString()); - transformAndSendEvent(properties); - } + protected abstract void transformAndSendEvent(HashMap properties); + @Override + public void onAfterTableRefreshed(AfterTableRefreshedEvent event) { + HashMap properties = new HashMap<>(); + properties.put("event_type", event.getClass().getSimpleName()); + properties.put("table_identifier", event.tableIdentifier().toString()); + transformAndSendEvent(properties); + } } diff --git a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index e94356f6b2..d1a95145da 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -36,8 +36,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.events.AfterTableRefreshedEvent; -import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.events.jsonEventListener.JsonEventListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/service/common/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/service/common/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index ef6e9467dc..51987502d2 100644 --- a/service/common/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -372,11 +372,12 @@ void handleSynchronousModeCorrectly(TestMode mode) { // Verify sync event assertThat(logEvents.events()) - .anySatisfy(logEvent -> { - String message = logEvent.message(); - assertThat(message).contains("test_table_sync"); - assertThat(message).contains("AfterTableRefreshedEvent"); - }); + .anySatisfy( + logEvent -> { + String message = logEvent.message(); + assertThat(message).contains("test_table_sync"); + assertThat(message).contains("AfterTableRefreshedEvent"); + }); } else { // Verify that putLogEvents was called with the expected content verify(client, times(1)) From 025de74256f0b1ad469a780aefca3f6bd783fbab Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 4 Aug 2025 14:14:05 -0700 Subject: [PATCH 19/39] revision on review from @singhpk234 --- .../QuarkusAwsCloudWatchConfiguration.java | 52 +++++++++++++++++++ .../jsonEventListener/JsonEventListener.java | 2 +- .../cloudwatch/Dockerfile-localstack-version | 1 - 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java index d7d56d739d..4d8dacb927 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java @@ -26,26 +26,78 @@ import java.util.Optional; import org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch.AwsCloudWatchConfiguration; +/** + * Quarkus-specific configuration interface for AWS CloudWatch event listener integration. + * + *

This interface extends the base {@link AwsCloudWatchConfiguration} and provides + * Quarkus-specific configuration mappings for AWS CloudWatch logging functionality.

+ * + */ @StaticInitSafe @ConfigMapping(prefix = "polaris.event-listener.aws-cloudwatch") @ApplicationScoped public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfiguration { + /** + * Returns the AWS CloudWatch log group name for event logging. + * + *

The log group is a collection of log streams that share the same retention, + * monitoring, and access control settings. If not specified, defaults to + * "polaris-cloudwatch-default-group".

+ * + *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-group}

+ * + * @return an Optional containing the log group name, or empty if not configured + */ @WithName("log-group") @WithDefault("polaris-cloudwatch-default-group") @Override Optional awsCloudwatchlogGroup(); + /** + * Returns the AWS CloudWatch log stream name for event logging. + * + *

A log stream is a sequence of log events that share the same source. + * Each log stream belongs to one log group. If not specified, defaults to + * "polaris-cloudwatch-default-stream".

+ * + *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-stream}

+ * + * @return an Optional containing the log stream name, or empty if not configured + */ @WithName("log-stream") @WithDefault("polaris-cloudwatch-default-stream") @Override Optional awsCloudwatchlogStream(); + /** + * Returns the AWS region where CloudWatch logs should be sent. + * + *

This specifies the AWS region for the CloudWatch service endpoint. + * The region must be a valid AWS region identifier. If not specified, + * defaults to "us-east-1".

+ * + *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.region}

+ * + * @return an Optional containing the AWS region, or empty if not configured + */ @WithName("region") @WithDefault("us-east-1") @Override Optional awsCloudwatchRegion(); + /** + * Returns the synchronous mode setting for CloudWatch logging. + * + *

When set to "true", log events are sent to CloudWatch synchronously, + * which may impact application performance but ensures immediate delivery. + * When set to "false" (default), log events are sent asynchronously for + * better performance.

+ * + *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.synchronous-mode}

+ * + * @return a String value ("true" or "false") indicating the synchronous mode setting + */ @WithName("synchronous-mode") @WithDefault("false") @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java index 5af945e92a..98f62194dd 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java @@ -33,4 +33,4 @@ public void onAfterTableRefreshed(AfterTableRefreshedEvent event) { properties.put("table_identifier", event.tableIdentifier().toString()); transformAndSendEvent(properties); } -} +} \ No newline at end of file diff --git a/service/common/src/test/resources/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/Dockerfile-localstack-version b/service/common/src/test/resources/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/Dockerfile-localstack-version index 2e17ab71e8..fd65524510 100644 --- a/service/common/src/test/resources/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/Dockerfile-localstack-version +++ b/service/common/src/test/resources/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/Dockerfile-localstack-version @@ -20,4 +20,3 @@ # Dockerfile to provide the image name and tag to a test. # Version is managed by Renovate - do not edit. FROM localstack/localstack:3.4 - From 491ea3ade5061ad3df9f87c9ab08a4f0b3afbf49 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 4 Aug 2025 14:16:25 -0700 Subject: [PATCH 20/39] resolve conflicts, pt. 2 --- .../org/apache/polaris/service/admin/PolarisServiceImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 966ed0a8ef..b4c1d58538 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -96,7 +96,6 @@ public class PolarisServiceImpl private final UserSecretsManagerFactory userSecretsManagerFactory; private final CallContext callContext; private final ReservedProperties reservedProperties; - private final PolarisEventListener polarisEventListener; @Inject public PolarisServiceImpl( @@ -113,7 +112,6 @@ public PolarisServiceImpl( this.polarisAuthorizer = polarisAuthorizer; this.callContext = callContext; this.reservedProperties = reservedProperties; - this.polarisEventListener = polarisEventListener; } private PolarisAdminService newAdminService( From d453660e67f1e5b4f9a5cffb5297715f07e8f3fc Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 4 Aug 2025 15:03:47 -0700 Subject: [PATCH 21/39] spotlessapply --- .../service/events/jsonEventListener/JsonEventListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java index 98f62194dd..5af945e92a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java @@ -33,4 +33,4 @@ public void onAfterTableRefreshed(AfterTableRefreshedEvent event) { properties.put("table_identifier", event.tableIdentifier().toString()); transformAndSendEvent(properties); } -} \ No newline at end of file +} From f89b0aeaa2fb1951789d2d885cc5e74c59d631c7 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 4 Aug 2025 15:23:47 -0700 Subject: [PATCH 22/39] spotlessapply again --- .../QuarkusAwsCloudWatchConfiguration.java | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java index 4d8dacb927..2ae9d9afd1 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java @@ -30,8 +30,7 @@ * Quarkus-specific configuration interface for AWS CloudWatch event listener integration. * *

This interface extends the base {@link AwsCloudWatchConfiguration} and provides - * Quarkus-specific configuration mappings for AWS CloudWatch logging functionality.

- * + * Quarkus-specific configuration mappings for AWS CloudWatch logging functionality. */ @StaticInitSafe @ConfigMapping(prefix = "polaris.event-listener.aws-cloudwatch") @@ -41,11 +40,10 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur /** * Returns the AWS CloudWatch log group name for event logging. * - *

The log group is a collection of log streams that share the same retention, - * monitoring, and access control settings. If not specified, defaults to - * "polaris-cloudwatch-default-group".

+ *

The log group is a collection of log streams that share the same retention, monitoring, and + * access control settings. If not specified, defaults to "polaris-cloudwatch-default-group". * - *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-group}

+ *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-group} * * @return an Optional containing the log group name, or empty if not configured */ @@ -57,11 +55,10 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur /** * Returns the AWS CloudWatch log stream name for event logging. * - *

A log stream is a sequence of log events that share the same source. - * Each log stream belongs to one log group. If not specified, defaults to - * "polaris-cloudwatch-default-stream".

+ *

A log stream is a sequence of log events that share the same source. Each log stream belongs + * to one log group. If not specified, defaults to "polaris-cloudwatch-default-stream". * - *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-stream}

+ *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-stream} * * @return an Optional containing the log stream name, or empty if not configured */ @@ -73,11 +70,10 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur /** * Returns the AWS region where CloudWatch logs should be sent. * - *

This specifies the AWS region for the CloudWatch service endpoint. - * The region must be a valid AWS region identifier. If not specified, - * defaults to "us-east-1".

+ *

This specifies the AWS region for the CloudWatch service endpoint. The region must be a + * valid AWS region identifier. If not specified, defaults to "us-east-1". * - *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.region}

+ *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.region} * * @return an Optional containing the AWS region, or empty if not configured */ @@ -89,12 +85,11 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur /** * Returns the synchronous mode setting for CloudWatch logging. * - *

When set to "true", log events are sent to CloudWatch synchronously, - * which may impact application performance but ensures immediate delivery. - * When set to "false" (default), log events are sent asynchronously for - * better performance.

+ *

When set to "true", log events are sent to CloudWatch synchronously, which may impact + * application performance but ensures immediate delivery. When set to "false" (default), log + * events are sent asynchronously for better performance. * - *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.synchronous-mode}

+ *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.synchronous-mode} * * @return a String value ("true" or "false") indicating the synchronous mode setting */ From e5c02b735cd847c4fc8a5b7cd43df7cdab5aedcd Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Wed, 6 Aug 2025 14:42:45 -0700 Subject: [PATCH 23/39] address comments from @RussellSpitzer --- .../aws/cloudwatch/AwsCloudWatchEventListener.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index d1a95145da..4a91771cd4 100644 --- a/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/service/common/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -220,6 +220,10 @@ private void reapFuturesMap() { if (future.isDone()) { Future.State currFutureState = future.state(); EventWithRetry currValue = futures.remove(future); + if (currValue == null) { + // This future was removed by some other thread and will be processed in that thread. + continue; + } if (currFutureState.equals(Future.State.FAILED) || currFutureState.equals(Future.State.CANCELLED)) { if (currValue.retryCount >= 3) { From 13053216f3d4686827286a172f3fe20497ab4aa1 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 12 Aug 2025 20:41:05 -0700 Subject: [PATCH 24/39] prior to manual test --- .../service/admin/PolarisServiceImpl.java | 3 +- .../AwsCloudWatchConfiguration.java | 2 +- .../AwsCloudWatchEventListener.java | 222 ++++++++---------- .../QuarkusAwsCloudWatchConfiguration.java | 4 +- .../service/admin/PolarisServiceImplTest.java | 3 +- .../AwsCloudWatchEventListenerTest.java | 122 +++++----- .../apache/polaris/service/TestServices.java | 3 +- 7 files changed, 166 insertions(+), 193 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index b4c1d58538..6301949e5e 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -104,8 +104,7 @@ public PolarisServiceImpl( UserSecretsManagerFactory userSecretsManagerFactory, PolarisAuthorizer polarisAuthorizer, CallContext callContext, - ReservedProperties reservedProperties, - PolarisEventListener polarisEventListener) { + ReservedProperties reservedProperties) { this.resolutionManifestFactory = resolutionManifestFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; this.userSecretsManagerFactory = userSecretsManagerFactory; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java index 5e6234b202..5302b66371 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java @@ -29,5 +29,5 @@ public interface AwsCloudWatchConfiguration { Optional awsCloudwatchRegion(); - String synchronousMode(); + boolean synchronousMode(); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index 4a91771cd4..bfb1bffd95 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Objects; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -41,9 +42,12 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsAsyncClient; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.DataAlreadyAcceptedException; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; @@ -61,14 +65,9 @@ public class AwsCloudWatchEventListener extends JsonEventListener { private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListener.class); private final ObjectMapper objectMapper = new ObjectMapper(); - private final ConcurrentHashMap, EventWithRetry> futures = new ConcurrentHashMap<>(); - record EventWithRetry(InputLogEvent inputLogEvent, int retryCount) {} - - private CloudWatchLogsClient client; - private volatile String sequenceToken; - - ExecutorService executorService; + private CloudWatchLogsAsyncClient client; +// private volatile String sequenceToken; private final String logGroup; private final String logStream; @@ -80,10 +79,7 @@ record EventWithRetry(InputLogEvent inputLogEvent, int retryCount) {} @Context SecurityContext securityContext; @Inject - public AwsCloudWatchEventListener( - AwsCloudWatchConfiguration config, ExecutorService executorService) { - this.executorService = executorService; - + public AwsCloudWatchEventListener(AwsCloudWatchConfiguration config) { this.logStream = config .awsCloudwatchlogStream() @@ -101,144 +97,112 @@ public AwsCloudWatchEventListener( .orElseThrow( () -> new IllegalArgumentException("AWS CloudWatch region must be configured"))); - this.synchronousMode = Boolean.parseBoolean(config.synchronousMode()); + this.synchronousMode = config.synchronousMode(); } @PostConstruct void start() { - this.client = createCloudWatchClient(); + this.client = createCloudWatchAsyncClient(); ensureLogGroupAndStream(); } - protected CloudWatchLogsClient createCloudWatchClient() { - return CloudWatchLogsClient.builder().region(region).build(); + protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { + return CloudWatchLogsAsyncClient.builder().region(region).build(); } private void ensureLogGroupAndStream() { try { - client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); + CompletableFuture future = client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); + future.join(); } catch (ResourceAlreadyExistsException ignored) { LOGGER.debug("Log group {} already exists", logGroup); } try { - client.createLogStream( + CompletableFuture future = client.createLogStream( CreateLogStreamRequest.builder().logGroupName(logGroup).logStreamName(logStream).build()); + future.join(); } catch (ResourceAlreadyExistsException ignored) { LOGGER.debug("Log stream {} already exists", logStream); } - sequenceToken = getSequenceToken(); +// sequenceToken = getSequenceToken(); } - private String getSequenceToken() { - DescribeLogStreamsResponse response = - client.describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(logGroup) - .logStreamNamePrefix(logStream) - .build()); - - return response.logStreams().stream() - .filter(s -> logStream.equals(s.logStreamName())) - .map(LogStream::uploadSequenceToken) - .filter(Objects::nonNull) - .findFirst() - .orElse(null); - } +// private String getSequenceToken() { +// DescribeLogStreamsResponse response = +// client.describeLogStreams( +// DescribeLogStreamsRequest.builder() +// .logGroupName(logGroup) +// .logStreamNamePrefix(logStream) +// .build()); +// +// return response.logStreams().stream() +// .filter(s -> logStream.equals(s.logStreamName())) +// .map(LogStream::uploadSequenceToken) +// .filter(Objects::nonNull) +// .findFirst() +// .orElse(null); +// } @PreDestroy void shutdown() { - for (Future future : futures.keySet()) { - if (!future.state().equals(Future.State.SUCCESS)) { - LOGGER.debug( - "Event not emitted to AWS CloudWatch due to being in state {}: {}", - future.state(), - futures.get(future).inputLogEvent); - future.cancel(true); - } - } if (client != null) { client.close(); } } - private void sendAndHandleCloudWatchCall(InputLogEvent event) { - try { - sendToCloudWatch(event); - } catch (DataAlreadyAcceptedException e) { - LOGGER.debug("Data already accepted: {}", e.getMessage()); - } catch (RuntimeException e) { - if (e instanceof SdkClientException - || e instanceof InvalidParameterException - || e instanceof UnrecognizedClientException) { - LOGGER.error( - "Error writing logs to CloudWatch - client-side error. Please adjust Polaris configurations: {}", - e.getMessage()); - } else { - LOGGER.error("Error writing logs to CloudWatch - server-side error: {}", e.getMessage()); - } - throw e; - } finally { - try { - reapFuturesMap(); - } catch (Exception e) { - LOGGER.debug("Futures map could not be reaped: {}", e.getMessage()); - } - } - } - - private void sendToCloudWatch(InputLogEvent event) { - PutLogEventsRequest.Builder requestBuilder = - PutLogEventsRequest.builder() - .logGroupName(logGroup) - .logStreamName(logStream) - .logEvents(List.of(event)); - - synchronized (this) { - if (sequenceToken != null) { - requestBuilder.sequenceToken(sequenceToken); - } - - try { - executePutLogEvents(requestBuilder); - } catch (InvalidSequenceTokenException e) { - sequenceToken = getSequenceToken(); - requestBuilder.sequenceToken(sequenceToken); - executePutLogEvents(requestBuilder); - } - } - } - - private void executePutLogEvents(PutLogEventsRequest.Builder requestBuilder) { - PutLogEventsResponse response = client.putLogEvents(requestBuilder.build()); - sequenceToken = response.nextSequenceToken(); - } - - private void reapFuturesMap() { - for (Future future : futures.keySet()) { - if (future.isDone()) { - Future.State currFutureState = future.state(); - EventWithRetry currValue = futures.remove(future); - if (currValue == null) { - // This future was removed by some other thread and will be processed in that thread. - continue; - } - if (currFutureState.equals(Future.State.FAILED) - || currFutureState.equals(Future.State.CANCELLED)) { - if (currValue.retryCount >= 3) { - LOGGER.error("Event retries failed. Event dropped: {}", currValue.inputLogEvent); - } else { - EventWithRetry newValue = - new EventWithRetry(currValue.inputLogEvent, currValue.retryCount + 1); - future = - executorService.submit(() -> sendAndHandleCloudWatchCall(newValue.inputLogEvent)); - futures.put(future, newValue); - } - } - } - } - } +// private void sendAndHandleCloudWatchCall(InputLogEvent event) { +// try { +// sendToCloudWatch(event); +// } catch (DataAlreadyAcceptedException e) { +// LOGGER.debug("Data already accepted: {}", e.getMessage()); +// } catch (RuntimeException e) { +// if (e instanceof SdkClientException +// || e instanceof InvalidParameterException +// || e instanceof UnrecognizedClientException) { +// LOGGER.error( +// "Error writing logs to CloudWatch - client-side error. Please adjust Polaris configurations. Event: {}, error: {}", +// event, e.getMessage()); +// } else { +// LOGGER.error("Error writing logs to CloudWatch - server-side error. Event: {}, error: {}", event, e.getMessage()); +// } +// throw e; +// } finally { +// try { +// reapFuturesMap(); +// } catch (Exception e) { +// LOGGER.debug("Futures map could not be reaped: {}", e.getMessage()); +// } +// } +// } + +// private void sendToCloudWatch(InputLogEvent event) { +// PutLogEventsRequest.Builder requestBuilder = +// PutLogEventsRequest.builder() +// .logGroupName(logGroup) +// .logStreamName(logStream) +// .logEvents(List.of(event)); +// +// synchronized (this) { +// if (sequenceToken != null) { +// requestBuilder.sequenceToken(sequenceToken); +// } +// +// try { +// executePutLogEvents(requestBuilder); +// } catch (InvalidSequenceTokenException e) { +// sequenceToken = getSequenceToken(); +// requestBuilder.sequenceToken(sequenceToken); +// executePutLogEvents(requestBuilder); +// } +// } +// } + +// private void executePutLogEvents(PutLogEventsRequest.Builder requestBuilder) { +// PutLogEventsResponse response = client.putLogEvents(requestBuilder.build()); +// sequenceToken = response.nextSequenceToken(); +// } @Override protected void transformAndSendEvent(HashMap properties) { @@ -253,11 +217,18 @@ protected void transformAndSendEvent(HashMap properties) { return; } InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp(callContext)); - if (!synchronousMode) { - Future future = executorService.submit(() -> sendAndHandleCloudWatchCall(inputLogEvent)); - futures.put(future, new EventWithRetry(inputLogEvent, 0)); - } else { - sendAndHandleCloudWatchCall(inputLogEvent); + PutLogEventsRequest.Builder requestBuilder = + PutLogEventsRequest.builder() + .logGroupName(logGroup) + .logStreamName(logStream) + .logEvents(List.of(inputLogEvent)); + CompletableFuture future = client.putLogEvents(requestBuilder.build()).whenComplete((resp, err) -> { + if (err != null) { + LOGGER.error("Error writing log to CloudWatch. Event: {}, Error: {}", inputLogEvent, err.getMessage()); + } + }); + if (synchronousMode) { + future.join(); } } @@ -268,9 +239,4 @@ private long getCurrentTimestamp(CallContext callContext) { private InputLogEvent createLogEvent(String eventAsJson, long timestamp) { return InputLogEvent.builder().message(eventAsJson).timestamp(timestamp).build(); } - - @VisibleForTesting - ConcurrentHashMap, EventWithRetry> getFutures() { - return futures; - } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java index 2ae9d9afd1..d088540ab6 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java @@ -91,10 +91,10 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur * *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.synchronous-mode} * - * @return a String value ("true" or "false") indicating the synchronous mode setting + * @return a boolean value indicating the synchronous mode setting */ @WithName("synchronous-mode") @WithDefault("false") @Override - String synchronousMode(); + boolean synchronousMode(); } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java index e0a205eb45..61e3d4efdc 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java @@ -81,8 +81,7 @@ void setUp() { userSecretsManagerFactory, polarisAuthorizer, callContext, - reservedProperties, - new NoOpPolarisEventListener()); + reservedProperties); } @Test diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 51987502d2..8758e01a6c 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -33,6 +33,7 @@ import java.time.Clock; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -57,6 +58,7 @@ import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsAsyncClient; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; @@ -108,7 +110,7 @@ void setUp() { when(config.awsCloudwatchlogGroup()).thenReturn(Optional.of(LOG_GROUP)); when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); - when(config.synchronousMode()).thenReturn("false"); // Default to async mode + when(config.synchronousMode()).thenReturn(false); // Default to async mode } @AfterEach @@ -127,13 +129,13 @@ void tearDown() throws Exception { } } - private CloudWatchLogsClient createCloudWatchClient(TestMode mode) { + private CloudWatchLogsAsyncClient createCloudWatchAsyncClient(TestMode mode) { switch (mode) { case LOCALSTACK: if (!localStack.isRunning()) { localStack.start(); } - return CloudWatchLogsClient.builder() + return CloudWatchLogsAsyncClient.builder() .endpointOverride(localStack.getEndpoint()) .credentialsProvider( StaticCredentialsProvider.create( @@ -142,11 +144,11 @@ private CloudWatchLogsClient createCloudWatchClient(TestMode mode) { .region(Region.of(localStack.getRegion())) .build(); case MOCKED: - CloudWatchLogsClient mockClient = Mockito.mock(CloudWatchLogsClient.class); + CloudWatchLogsAsyncClient mockClient = Mockito.mock(CloudWatchLogsAsyncClient.class); // Mock the responses for log group and stream creation - when(mockClient.createLogGroup(any(CreateLogGroupRequest.class))).thenReturn(null); - when(mockClient.createLogStream(any(CreateLogStreamRequest.class))).thenReturn(null); + when(mockClient.createLogGroup(any(CreateLogGroupRequest.class))).thenReturn(CompletableFuture.completedFuture(null)); + when(mockClient.createLogStream(any(CreateLogStreamRequest.class))).thenReturn(CompletableFuture.completedFuture(null)); // Mock the describe log streams response for getting sequence token DescribeLogStreamsResponse mockStreamsResponse = @@ -158,12 +160,12 @@ private CloudWatchLogsClient createCloudWatchClient(TestMode mode) { .build()) .build(); when(mockClient.describeLogStreams(any(DescribeLogStreamsRequest.class))) - .thenReturn(mockStreamsResponse); + .thenReturn(CompletableFuture.completedFuture(mockStreamsResponse)); // Mock the putLogEvents response PutLogEventsResponse mockPutResponse = PutLogEventsResponse.builder().nextSequenceToken("mock-sequence-token-123").build(); - when(mockClient.putLogEvents(any(PutLogEventsRequest.class))).thenReturn(mockPutResponse); + when(mockClient.putLogEvents(any(PutLogEventsRequest.class))).thenReturn(CompletableFuture.completedFuture(mockPutResponse)); return mockClient; default: @@ -171,11 +173,11 @@ private CloudWatchLogsClient createCloudWatchClient(TestMode mode) { } } - private AwsCloudWatchEventListener createListener(CloudWatchLogsClient client) { + private AwsCloudWatchEventListener createListener(CloudWatchLogsAsyncClient client) { AwsCloudWatchEventListener listener = - new AwsCloudWatchEventListener(config, executorService) { + new AwsCloudWatchEventListener(config) { @Override - protected CloudWatchLogsClient createCloudWatchClient() { + protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { return client; } }; @@ -198,24 +200,24 @@ protected CloudWatchLogsClient createCloudWatchClient() { return listener; } - private void waitForFuture(Future future, long timeoutMs) { - try { - long start = System.currentTimeMillis(); - while (!future.isDone()) { - if (System.currentTimeMillis() - start > timeoutMs) { - fail("Future did not complete in time"); - } - Thread.sleep(100); - } - } catch (InterruptedException e) { - fail("Future was interrupted"); - } - } +// private void waitForFuture(Future future, long timeoutMs) { +// try { +// long start = System.currentTimeMillis(); +// while (!future.isDone()) { +// if (System.currentTimeMillis() - start > timeoutMs) { +// fail("Future did not complete in time"); +// } +// Thread.sleep(100); +// } +// } catch (InterruptedException e) { +// fail("Future was interrupted"); +// } +// } @ParameterizedTest @MethodSource("testModeProvider") void shouldCreateLogGroupAndStream(TestMode mode) { - CloudWatchLogsClient client = createCloudWatchClient(mode); + CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(mode); AwsCloudWatchEventListener listener = createListener(client); // Start the listener which should create the log group and stream @@ -225,7 +227,7 @@ void shouldCreateLogGroupAndStream(TestMode mode) { // Verify log group exists DescribeLogGroupsResponse groups = client.describeLogGroups( - DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()); + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()).join(); assertThat(groups.logGroups()) .hasSize(1) .first() @@ -237,7 +239,7 @@ void shouldCreateLogGroupAndStream(TestMode mode) { DescribeLogStreamsRequest.builder() .logGroupName(LOG_GROUP) .logStreamNamePrefix(LOG_STREAM) - .build()); + .build()).join(); assertThat(streams.logStreams()) .hasSize(1) .first() @@ -253,12 +255,12 @@ void shouldCreateLogGroupAndStream(TestMode mode) { (CreateLogStreamRequest request) -> request.logGroupName().equals(LOG_GROUP) && request.logStreamName().equals(LOG_STREAM))); - verify(client, times(1)) - .describeLogStreams( - argThat( - (DescribeLogStreamsRequest request) -> - request.logGroupName().equals(LOG_GROUP) - && request.logStreamNamePrefix().equals(LOG_STREAM))); +// verify(client, times(1)) +// .describeLogStreams( +// argThat( +// (DescribeLogStreamsRequest request) -> +// request.logGroupName().equals(LOG_GROUP) +// && request.logStreamNamePrefix().equals(LOG_STREAM))); } // Clean up @@ -271,7 +273,7 @@ void shouldCreateLogGroupAndStream(TestMode mode) { @ParameterizedTest @MethodSource("testModeProvider") void shouldSendEventToCloudWatch(TestMode mode) { - CloudWatchLogsClient client = createCloudWatchClient(mode); + CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(mode); AwsCloudWatchEventListener listener = createListener(client); // Start the listener @@ -282,26 +284,35 @@ void shouldSendEventToCloudWatch(TestMode mode) { AfterTableRefreshedEvent event = new AfterTableRefreshedEvent(testTable); listener.onAfterTableRefreshed(event); - // Wait for the future to complete - List> activeFutures = listener.getFutures().keySet().stream().toList(); - assertThat(activeFutures).hasSize(1); - Future eventFuture = activeFutures.getFirst(); - - long timeout = mode == TestMode.MOCKED ? 5000L : 30000L; // Shorter timeout for mocked tests - waitForFuture(eventFuture, timeout); - - // Verify the future completed successfully - assertThat(eventFuture.isDone()).isTrue(); - assertThat(eventFuture.state()).isEqualTo(Future.State.SUCCESS); - if (mode == TestMode.LOCALSTACK) { // Verify the event was sent to CloudWatch by reading the actual logs - GetLogEventsResponse logEvents = - client.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()); + GetLogEventsResponse logEvents; + int eventCount = 0; + long startTime = System.currentTimeMillis(); + while (eventCount == 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + fail("Thread interrupted waiting for event"); + } + if (System.currentTimeMillis() - startTime > 30_000) { + fail("Timeout exceeded while waiting for event to arrive"); + } + logEvents = + client.getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()).join(); + eventCount = logEvents.events().size(); + } + + logEvents = + client.getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()).join(); assertThat(logEvents.events()) .hasSize(1) @@ -346,10 +357,10 @@ void shouldSendEventToCloudWatch(TestMode mode) { @ParameterizedTest @MethodSource("testModeProvider") void handleSynchronousModeCorrectly(TestMode mode) { - CloudWatchLogsClient client = createCloudWatchClient(mode); + CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(mode); // Test synchronous mode - when(config.synchronousMode()).thenReturn("true"); + when(config.synchronousMode()).thenReturn(true); AwsCloudWatchEventListener syncListener = createListener(client); syncListener.start(); @@ -357,7 +368,6 @@ void handleSynchronousModeCorrectly(TestMode mode) { TableIdentifier syncTestTable = TableIdentifier.of("test_namespace", "test_table_sync"); AfterTableRefreshedEvent syncEvent = new AfterTableRefreshedEvent(syncTestTable); syncListener.onAfterTableRefreshed(syncEvent); - assertThat(syncListener.getFutures()).isEmpty(); // No futures should be created in sync mode if (mode == TestMode.LOCALSTACK) { // Verify both events were sent to CloudWatch @@ -366,7 +376,7 @@ void handleSynchronousModeCorrectly(TestMode mode) { GetLogEventsRequest.builder() .logGroupName(LOG_GROUP) .logStreamName(LOG_STREAM) - .build()); + .build()).join(); assertThat(logEvents.events()).hasSize(1); diff --git a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 5f0326e30d..ed78d8bfe4 100644 --- a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -278,8 +278,7 @@ public String getAuthenticationScheme() { userSecretsManagerFactory, authorizer, callContext, - reservedProperties, - polarisEventListener)); + reservedProperties)); return new TestServices( catalogsApi, From 27f28f401f3c4ada4e1e3b56f9b9b5516694dff8 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 12 Aug 2025 20:58:05 -0700 Subject: [PATCH 25/39] addressing comments from @snazy --- .../AwsCloudWatchEventListener.java | 104 +++--------------- .../AwsCloudWatchEventListenerTest.java | 23 ---- 2 files changed, 14 insertions(+), 113 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index bfb1bffd95..3adec433e1 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; @@ -31,34 +30,22 @@ import jakarta.ws.rs.core.SecurityContext; import java.util.HashMap; import java.util.List; -import java.util.Objects; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.concurrent.CompletionException; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.service.events.jsonEventListener.JsonEventListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsAsyncClient; -import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamResponse; -import software.amazon.awssdk.services.cloudwatchlogs.model.DataAlreadyAcceptedException; -import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; -import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.InputLogEvent; -import software.amazon.awssdk.services.cloudwatchlogs.model.InvalidParameterException; -import software.amazon.awssdk.services.cloudwatchlogs.model.InvalidSequenceTokenException; -import software.amazon.awssdk.services.cloudwatchlogs.model.LogStream; import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.ResourceAlreadyExistsException; -import software.amazon.awssdk.services.cloudwatchlogs.model.UnrecognizedClientException; @ApplicationScoped @Identifier("aws-cloudwatch") @@ -67,7 +54,6 @@ public class AwsCloudWatchEventListener extends JsonEventListener { private final ObjectMapper objectMapper = new ObjectMapper(); private CloudWatchLogsAsyncClient client; -// private volatile String sequenceToken; private final String logGroup; private final String logStream; @@ -114,37 +100,27 @@ private void ensureLogGroupAndStream() { try { CompletableFuture future = client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); future.join(); - } catch (ResourceAlreadyExistsException ignored) { - LOGGER.debug("Log group {} already exists", logGroup); + } catch (CompletionException e) { + if (e.getCause() instanceof ResourceAlreadyExistsException) { + LOGGER.debug("Log group {} already exists", logGroup); + } else { + throw e; + } } try { CompletableFuture future = client.createLogStream( - CreateLogStreamRequest.builder().logGroupName(logGroup).logStreamName(logStream).build()); + CreateLogStreamRequest.builder().logGroupName(logGroup).logStreamName(logStream).build()); future.join(); - } catch (ResourceAlreadyExistsException ignored) { - LOGGER.debug("Log stream {} already exists", logStream); + } catch (CompletionException e) { + if (e.getCause() instanceof ResourceAlreadyExistsException) { + LOGGER.debug("Log stream {} already exists", logStream); + } else { + throw e; + } } - -// sequenceToken = getSequenceToken(); } -// private String getSequenceToken() { -// DescribeLogStreamsResponse response = -// client.describeLogStreams( -// DescribeLogStreamsRequest.builder() -// .logGroupName(logGroup) -// .logStreamNamePrefix(logStream) -// .build()); -// -// return response.logStreams().stream() -// .filter(s -> logStream.equals(s.logStreamName())) -// .map(LogStream::uploadSequenceToken) -// .filter(Objects::nonNull) -// .findFirst() -// .orElse(null); -// } - @PreDestroy void shutdown() { if (client != null) { @@ -152,58 +128,6 @@ void shutdown() { } } -// private void sendAndHandleCloudWatchCall(InputLogEvent event) { -// try { -// sendToCloudWatch(event); -// } catch (DataAlreadyAcceptedException e) { -// LOGGER.debug("Data already accepted: {}", e.getMessage()); -// } catch (RuntimeException e) { -// if (e instanceof SdkClientException -// || e instanceof InvalidParameterException -// || e instanceof UnrecognizedClientException) { -// LOGGER.error( -// "Error writing logs to CloudWatch - client-side error. Please adjust Polaris configurations. Event: {}, error: {}", -// event, e.getMessage()); -// } else { -// LOGGER.error("Error writing logs to CloudWatch - server-side error. Event: {}, error: {}", event, e.getMessage()); -// } -// throw e; -// } finally { -// try { -// reapFuturesMap(); -// } catch (Exception e) { -// LOGGER.debug("Futures map could not be reaped: {}", e.getMessage()); -// } -// } -// } - -// private void sendToCloudWatch(InputLogEvent event) { -// PutLogEventsRequest.Builder requestBuilder = -// PutLogEventsRequest.builder() -// .logGroupName(logGroup) -// .logStreamName(logStream) -// .logEvents(List.of(event)); -// -// synchronized (this) { -// if (sequenceToken != null) { -// requestBuilder.sequenceToken(sequenceToken); -// } -// -// try { -// executePutLogEvents(requestBuilder); -// } catch (InvalidSequenceTokenException e) { -// sequenceToken = getSequenceToken(); -// requestBuilder.sequenceToken(sequenceToken); -// executePutLogEvents(requestBuilder); -// } -// } -// } - -// private void executePutLogEvents(PutLogEventsRequest.Builder requestBuilder) { -// PutLogEventsResponse response = client.putLogEvents(requestBuilder.build()); -// sequenceToken = response.nextSequenceToken(); -// } - @Override protected void transformAndSendEvent(HashMap properties) { properties.put("realm", callContext.getRealmContext().getRealmIdentifier()); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 8758e01a6c..e507988342 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -31,12 +31,10 @@ import jakarta.ws.rs.core.SecurityContext; import java.security.Principal; import java.time.Clock; -import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.apache.iceberg.catalog.TableIdentifier; @@ -59,7 +57,6 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsAsyncClient; -import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsRequest; @@ -200,20 +197,6 @@ protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { return listener; } -// private void waitForFuture(Future future, long timeoutMs) { -// try { -// long start = System.currentTimeMillis(); -// while (!future.isDone()) { -// if (System.currentTimeMillis() - start > timeoutMs) { -// fail("Future did not complete in time"); -// } -// Thread.sleep(100); -// } -// } catch (InterruptedException e) { -// fail("Future was interrupted"); -// } -// } - @ParameterizedTest @MethodSource("testModeProvider") void shouldCreateLogGroupAndStream(TestMode mode) { @@ -255,12 +238,6 @@ void shouldCreateLogGroupAndStream(TestMode mode) { (CreateLogStreamRequest request) -> request.logGroupName().equals(LOG_GROUP) && request.logStreamName().equals(LOG_STREAM))); -// verify(client, times(1)) -// .describeLogStreams( -// argThat( -// (DescribeLogStreamsRequest request) -> -// request.logGroupName().equals(LOG_GROUP) -// && request.logStreamNamePrefix().equals(LOG_STREAM))); } // Clean up From ec2bee892bf3c98c5086286c69a0c8032bbaf96b Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 12 Aug 2025 21:27:42 -0700 Subject: [PATCH 26/39] merge from main --- .../AwsCloudWatchEventListener.java | 12 ++++++--- .../AwsCloudWatchEventListenerTest.java | 27 ++++++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index 3adec433e1..1787c6b4da 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -28,6 +28,8 @@ import jakarta.inject.Inject; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.SecurityContext; + +import java.time.Clock; import java.util.HashMap; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -59,13 +61,14 @@ public class AwsCloudWatchEventListener extends JsonEventListener { private final String logStream; private final Region region; private final boolean synchronousMode; + private final Clock clock; @Inject CallContext callContext; @Context SecurityContext securityContext; @Inject - public AwsCloudWatchEventListener(AwsCloudWatchConfiguration config) { + public AwsCloudWatchEventListener(AwsCloudWatchConfiguration config, Clock clock) { this.logStream = config .awsCloudwatchlogStream() @@ -84,6 +87,7 @@ public AwsCloudWatchEventListener(AwsCloudWatchConfiguration config) { () -> new IllegalArgumentException("AWS CloudWatch region must be configured"))); this.synchronousMode = config.synchronousMode(); + this.clock = clock; } @PostConstruct @@ -140,7 +144,7 @@ protected void transformAndSendEvent(HashMap properties) { LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); return; } - InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp(callContext)); + InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp()); PutLogEventsRequest.Builder requestBuilder = PutLogEventsRequest.builder() .logGroupName(logGroup) @@ -156,8 +160,8 @@ protected void transformAndSendEvent(HashMap properties) { } } - private long getCurrentTimestamp(CallContext callContext) { - return callContext.getPolarisCallContext().getClock().millis(); + private long getCurrentTimestamp() { + return clock.millis(); } private InputLogEvent createLogEvent(String eventAsJson, long timestamp) { diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index e507988342..26ba9fc594 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -82,6 +82,7 @@ class AwsCloudWatchEventListenerTest { private static final String LOG_STREAM = "test-log-stream"; private static final String REALM = "test-realm"; private static final String TEST_USER = "test-user"; + private static final Clock clock = Clock.systemUTC(); @Mock private AwsCloudWatchConfiguration config; @@ -172,7 +173,7 @@ private CloudWatchLogsAsyncClient createCloudWatchAsyncClient(TestMode mode) { private AwsCloudWatchEventListener createListener(CloudWatchLogsAsyncClient client) { AwsCloudWatchEventListener listener = - new AwsCloudWatchEventListener(config) { + new AwsCloudWatchEventListener(config, clock) { @Override protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { return client; @@ -187,7 +188,6 @@ protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { Principal principal = Mockito.mock(Principal.class); when(callContext.getRealmContext()).thenReturn(realmContext); when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); - when(polarisCallContext.getClock()).thenReturn(Clock.systemUTC()); when(realmContext.getRealmIdentifier()).thenReturn(REALM); when(securityContext.getUserPrincipal()).thenReturn(principal); when(principal.getName()).thenReturn(TEST_USER); @@ -348,7 +348,28 @@ void handleSynchronousModeCorrectly(TestMode mode) { if (mode == TestMode.LOCALSTACK) { // Verify both events were sent to CloudWatch - GetLogEventsResponse logEvents = + GetLogEventsResponse logEvents; + int eventCount = 0; + long startTime = System.currentTimeMillis(); + while (eventCount == 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + fail("Thread interrupted waiting for event"); + } + if (System.currentTimeMillis() - startTime > 30_000) { + fail("Timeout exceeded while waiting for event to arrive"); + } + logEvents = + client.getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()).join(); + eventCount = logEvents.events().size(); + } + + logEvents = client.getLogEvents( GetLogEventsRequest.builder() .logGroupName(LOG_GROUP) From 69b7feb58da9b6cbc242b8b8f6d10090d93dbcca Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 12 Aug 2025 21:49:59 -0700 Subject: [PATCH 27/39] spotlesscheck --- .../service/admin/PolarisServiceImpl.java | 1 - .../AwsCloudWatchEventListener.java | 37 +++++---- .../service/admin/PolarisServiceImplTest.java | 1 - .../AwsCloudWatchEventListenerTest.java | 77 +++++++++++-------- 4 files changed, 70 insertions(+), 46 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 90bfbd1f1a..5318c00b8f 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -78,7 +78,6 @@ import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; import org.apache.polaris.service.config.ReservedProperties; -import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.types.PolicyIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index 1787c6b4da..dd80c64d6d 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -28,7 +28,6 @@ import jakarta.inject.Inject; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.SecurityContext; - import java.time.Clock; import java.util.HashMap; import java.util.List; @@ -102,7 +101,8 @@ protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { private void ensureLogGroupAndStream() { try { - CompletableFuture future = client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); + CompletableFuture future = + client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); future.join(); } catch (CompletionException e) { if (e.getCause() instanceof ResourceAlreadyExistsException) { @@ -113,8 +113,12 @@ private void ensureLogGroupAndStream() { } try { - CompletableFuture future = client.createLogStream( - CreateLogStreamRequest.builder().logGroupName(logGroup).logStreamName(logStream).build()); + CompletableFuture future = + client.createLogStream( + CreateLogStreamRequest.builder() + .logGroupName(logGroup) + .logStreamName(logStream) + .build()); future.join(); } catch (CompletionException e) { if (e.getCause() instanceof ResourceAlreadyExistsException) { @@ -146,15 +150,22 @@ protected void transformAndSendEvent(HashMap properties) { } InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp()); PutLogEventsRequest.Builder requestBuilder = - PutLogEventsRequest.builder() - .logGroupName(logGroup) - .logStreamName(logStream) - .logEvents(List.of(inputLogEvent)); - CompletableFuture future = client.putLogEvents(requestBuilder.build()).whenComplete((resp, err) -> { - if (err != null) { - LOGGER.error("Error writing log to CloudWatch. Event: {}, Error: {}", inputLogEvent, err.getMessage()); - } - }); + PutLogEventsRequest.builder() + .logGroupName(logGroup) + .logStreamName(logStream) + .logEvents(List.of(inputLogEvent)); + CompletableFuture future = + client + .putLogEvents(requestBuilder.build()) + .whenComplete( + (resp, err) -> { + if (err != null) { + LOGGER.error( + "Error writing log to CloudWatch. Event: {}, Error: {}", + inputLogEvent, + err.getMessage()); + } + }); if (synchronousMode) { future.join(); } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java index 61e3d4efdc..75adb59f5b 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisServiceImplTest.java @@ -40,7 +40,6 @@ import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.config.ReservedProperties; -import org.apache.polaris.service.events.NoOpPolarisEventListener; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 26ba9fc594..147532f88d 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -145,8 +145,10 @@ private CloudWatchLogsAsyncClient createCloudWatchAsyncClient(TestMode mode) { CloudWatchLogsAsyncClient mockClient = Mockito.mock(CloudWatchLogsAsyncClient.class); // Mock the responses for log group and stream creation - when(mockClient.createLogGroup(any(CreateLogGroupRequest.class))).thenReturn(CompletableFuture.completedFuture(null)); - when(mockClient.createLogStream(any(CreateLogStreamRequest.class))).thenReturn(CompletableFuture.completedFuture(null)); + when(mockClient.createLogGroup(any(CreateLogGroupRequest.class))) + .thenReturn(CompletableFuture.completedFuture(null)); + when(mockClient.createLogStream(any(CreateLogStreamRequest.class))) + .thenReturn(CompletableFuture.completedFuture(null)); // Mock the describe log streams response for getting sequence token DescribeLogStreamsResponse mockStreamsResponse = @@ -163,7 +165,8 @@ private CloudWatchLogsAsyncClient createCloudWatchAsyncClient(TestMode mode) { // Mock the putLogEvents response PutLogEventsResponse mockPutResponse = PutLogEventsResponse.builder().nextSequenceToken("mock-sequence-token-123").build(); - when(mockClient.putLogEvents(any(PutLogEventsRequest.class))).thenReturn(CompletableFuture.completedFuture(mockPutResponse)); + when(mockClient.putLogEvents(any(PutLogEventsRequest.class))) + .thenReturn(CompletableFuture.completedFuture(mockPutResponse)); return mockClient; default: @@ -209,8 +212,10 @@ void shouldCreateLogGroupAndStream(TestMode mode) { if (mode == TestMode.LOCALSTACK) { // Verify log group exists DescribeLogGroupsResponse groups = - client.describeLogGroups( - DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()).join(); + client + .describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) + .join(); assertThat(groups.logGroups()) .hasSize(1) .first() @@ -218,11 +223,13 @@ void shouldCreateLogGroupAndStream(TestMode mode) { // Verify log stream exists DescribeLogStreamsResponse streams = - client.describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamNamePrefix(LOG_STREAM) - .build()).join(); + client + .describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamNamePrefix(LOG_STREAM) + .build()) + .join(); assertThat(streams.logStreams()) .hasSize(1) .first() @@ -275,21 +282,25 @@ void shouldSendEventToCloudWatch(TestMode mode) { if (System.currentTimeMillis() - startTime > 30_000) { fail("Timeout exceeded while waiting for event to arrive"); } - logEvents = - client.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()).join(); + logEvents = + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); eventCount = logEvents.events().size(); } logEvents = - client.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()).join(); + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); assertThat(logEvents.events()) .hasSize(1) @@ -361,20 +372,24 @@ void handleSynchronousModeCorrectly(TestMode mode) { fail("Timeout exceeded while waiting for event to arrive"); } logEvents = - client.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()).join(); + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); eventCount = logEvents.events().size(); } logEvents = - client.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()).join(); + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); assertThat(logEvents.events()).hasSize(1); From e8b5e93ea56ce53ecced6bd382253e0c519cdcf7 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 12 Aug 2025 23:03:08 -0700 Subject: [PATCH 28/39] documentation updates --- .../events/jsonEventListener/JsonEventListener.java | 9 +++++++++ site/content/in-dev/unreleased/configuration.md | 11 ++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java index 5af945e92a..be0123f808 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java @@ -23,6 +23,15 @@ import org.apache.polaris.service.events.AfterTableRefreshedEvent; import org.apache.polaris.service.events.PolarisEventListener; +/** + * Abstract base class from which all event sinks that output events in JSON format can extend. + *

+ * This class provides a common framework for transforming Polaris events into JSON format + * and sending them to various destinations. Concrete implementations should override the + * {@link #transformAndSendEvent(HashMap)} method to define how the JSON event data should + * be transmitted or stored. + *

+ */ public abstract class JsonEventListener extends PolarisEventListener { protected abstract void transformAndSendEvent(HashMap properties); diff --git a/site/content/in-dev/unreleased/configuration.md b/site/content/in-dev/unreleased/configuration.md index 750d808219..5e0ba3f4f6 100644 --- a/site/content/in-dev/unreleased/configuration.md +++ b/site/content/in-dev/unreleased/configuration.md @@ -117,11 +117,12 @@ read-only mode, as Polaris only reads the configuration file once, at startup. | `polaris.metrics.realm-id-tag.http-metrics-max-cardinality` | `100` | The maximum cardinality for the `realm_id` tag in HTTP request metrics. | | `polaris.tasks.max-concurrent-tasks` | `100` | Define the max number of concurrent tasks. | | `polaris.tasks.max-queued-tasks` | `1000` | Define the max number of tasks in queue. | - | `polaris.config.rollback.compaction.on-conflicts.enabled` | `false` | When set to true Polaris will apply the deconfliction by rollbacking those REPLACE operations snapshots which have the property of `polaris.internal.rollback.compaction.on-conflict` in their snapshot summary set to `rollback`, to resolve conflicts at the server end. | -| `polaris.event-listener.type` | `no-op` | Define the Polaris event listener type. Supported values are `no-op`, `aws-cloudwatch`. | -| `polaris.event-listener.aws-cloudwatch.log-group` | | Define the AWS CloudWatch log group name for the event listener. | -| `polaris.event-listener.aws-cloudwatch.log-stream` | | Define the AWS CloudWatch log stream name for the event listener. | -| `polaris.event-listener.aws-cloudwatch.region` | | Define the AWS region for the CloudWatch event listener. | + | `polaris.config.rollback.compaction.on-conflicts.enabled` | `false` | When set to true Polaris will apply the deconfliction by rollbacking those REPLACE operations snapshots which have the property of `polaris.internal.rollback.compaction.on-conflict` in their snapshot summary set to `rollback`, to resolve conflicts at the server end. | +| `polaris.event-listener.type` | `no-op` | Define the Polaris event listener type. Supported values are `no-op`, `aws-cloudwatch`. | +| `polaris.event-listener.aws-cloudwatch.log-group` | | Define the AWS CloudWatch log group name for the event listener. | +| `polaris.event-listener.aws-cloudwatch.log-stream` | | Define the AWS CloudWatch log stream name for the event listener. | +| `polaris.event-listener.aws-cloudwatch.region` | | Define the AWS region for the CloudWatch event listener. | +| `polaris.event-listener.aws-cloudwatch.synchronous-mode` | `false` | Define whether log events are sent to CloudWatch synchronously. When set to true, events are sent synchronously which may impact performance but ensures immediate delivery. When false (default), events are sent asynchronously for better performance. | There are non Polaris configuration properties that can be useful: From 3030d6ab00c26215b86c4b37343149010b11d282 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Wed, 13 Aug 2025 15:39:13 -0700 Subject: [PATCH 29/39] review comments from @RussellSpitzer --- .../jsonEventListener/JsonEventListener.java | 11 ++++---- .../AwsCloudWatchConfiguration.java | 8 +++--- .../AwsCloudWatchEventListener.java | 26 ++++--------------- .../QuarkusAwsCloudWatchConfiguration.java | 7 +++-- .../AwsCloudWatchEventListenerTest.java | 7 +++-- 5 files changed, 19 insertions(+), 40 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java index be0123f808..4e4ab65e3c 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java @@ -25,12 +25,11 @@ /** * Abstract base class from which all event sinks that output events in JSON format can extend. - *

- * This class provides a common framework for transforming Polaris events into JSON format - * and sending them to various destinations. Concrete implementations should override the - * {@link #transformAndSendEvent(HashMap)} method to define how the JSON event data should - * be transmitted or stored. - *

+ * + *

This class provides a common framework for transforming Polaris events into JSON format and + * sending them to various destinations. Concrete implementations should override the {@link + * #transformAndSendEvent(HashMap)} method to define how the JSON event data should be transmitted + * or stored. */ public abstract class JsonEventListener extends PolarisEventListener { protected abstract void transformAndSendEvent(HashMap properties); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java index 5302b66371..64f55fba8f 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java @@ -19,15 +19,13 @@ package org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch; -import java.util.Optional; - /** Configuration interface for AWS CloudWatch event listener settings. */ public interface AwsCloudWatchConfiguration { - Optional awsCloudwatchlogGroup(); + String awsCloudwatchlogGroup(); - Optional awsCloudwatchlogStream(); + String awsCloudwatchlogStream(); - Optional awsCloudwatchRegion(); + String awsCloudwatchRegion(); boolean synchronousMode(); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index dd80c64d6d..8828ba4809 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -68,23 +68,9 @@ public class AwsCloudWatchEventListener extends JsonEventListener { @Inject public AwsCloudWatchEventListener(AwsCloudWatchConfiguration config, Clock clock) { - this.logStream = - config - .awsCloudwatchlogStream() - .orElseThrow( - () -> new IllegalArgumentException("AWS CloudWatch log stream must be configured")); - this.logGroup = - config - .awsCloudwatchlogGroup() - .orElseThrow( - () -> new IllegalArgumentException("AWS CloudWatch log group must be configured")); - this.region = - Region.of( - config - .awsCloudwatchRegion() - .orElseThrow( - () -> - new IllegalArgumentException("AWS CloudWatch region must be configured"))); + this.logStream = config.awsCloudwatchlogStream(); + this.logGroup = config.awsCloudwatchlogGroup(); + this.region = Region.of(config.awsCloudwatchRegion()); this.synchronousMode = config.synchronousMode(); this.clock = clock; } @@ -145,7 +131,7 @@ protected void transformAndSendEvent(HashMap properties) { try { eventAsJson = objectMapper.writeValueAsString(properties); } catch (JsonProcessingException e) { - LOGGER.error("Error processing event into JSON string: {}", e.getMessage()); + LOGGER.error("Error processing event into JSON string: ", e); return; } InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp()); @@ -161,9 +147,7 @@ protected void transformAndSendEvent(HashMap properties) { (resp, err) -> { if (err != null) { LOGGER.error( - "Error writing log to CloudWatch. Event: {}, Error: {}", - inputLogEvent, - err.getMessage()); + "Error writing log to CloudWatch. Event: {}, Error: ", inputLogEvent, err); } }); if (synchronousMode) { diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java index d088540ab6..eca04d1ecb 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java @@ -23,7 +23,6 @@ import io.smallrye.config.WithDefault; import io.smallrye.config.WithName; import jakarta.enterprise.context.ApplicationScoped; -import java.util.Optional; import org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch.AwsCloudWatchConfiguration; /** @@ -50,7 +49,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur @WithName("log-group") @WithDefault("polaris-cloudwatch-default-group") @Override - Optional awsCloudwatchlogGroup(); + String awsCloudwatchlogGroup(); /** * Returns the AWS CloudWatch log stream name for event logging. @@ -65,7 +64,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur @WithName("log-stream") @WithDefault("polaris-cloudwatch-default-stream") @Override - Optional awsCloudwatchlogStream(); + String awsCloudwatchlogStream(); /** * Returns the AWS region where CloudWatch logs should be sent. @@ -80,7 +79,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur @WithName("region") @WithDefault("us-east-1") @Override - Optional awsCloudwatchRegion(); + String awsCloudwatchRegion(); /** * Returns the synchronous mode setting for CloudWatch logging. diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 147532f88d..238090b6f9 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -31,7 +31,6 @@ import jakarta.ws.rs.core.SecurityContext; import java.security.Principal; import java.time.Clock; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -105,9 +104,9 @@ void setUp() { executorService = Executors.newSingleThreadExecutor(); // Configure the mocks - when(config.awsCloudwatchlogGroup()).thenReturn(Optional.of(LOG_GROUP)); - when(config.awsCloudwatchlogStream()).thenReturn(Optional.of(LOG_STREAM)); - when(config.awsCloudwatchRegion()).thenReturn(Optional.of("us-east-1")); + when(config.awsCloudwatchlogGroup()).thenReturn(LOG_GROUP); + when(config.awsCloudwatchlogStream()).thenReturn(LOG_STREAM); + when(config.awsCloudwatchRegion()).thenReturn("us-east-1"); when(config.synchronousMode()).thenReturn(false); // Default to async mode } From b9abab6ebf8234e6444bbeeb636b70f8a6864114 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Wed, 13 Aug 2025 15:54:54 -0700 Subject: [PATCH 30/39] removed mocked tests, as per review from @RussellSpitzer --- .../AwsCloudWatchEventListenerTest.java | 351 ++++++------------ 1 file changed, 115 insertions(+), 236 deletions(-) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 238090b6f9..fda3eb6b27 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -22,20 +22,14 @@ import static org.apache.polaris.containerspec.ContainerSpecHelper.containerSpecHelper; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import jakarta.ws.rs.core.SecurityContext; import java.security.Principal; import java.time.Clock; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; @@ -43,9 +37,7 @@ import org.apache.polaris.service.events.AfterTableRefreshedEvent; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; @@ -56,16 +48,12 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsAsyncClient; -import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; -import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsResponse; -import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsRequest; -import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsResponse; class AwsCloudWatchEventListenerTest { private static final Logger LOGGER = @@ -88,16 +76,6 @@ class AwsCloudWatchEventListenerTest { private ExecutorService executorService; private AutoCloseable mockitoContext; - enum TestMode { - LOCALSTACK, - MOCKED - } - - // Test data provider - static Stream testModeProvider() { - return Stream.of(Arguments.of(TestMode.LOCALSTACK), Arguments.of(TestMode.MOCKED)); - } - @BeforeEach void setUp() { mockitoContext = MockitoAnnotations.openMocks(this); @@ -126,51 +104,17 @@ void tearDown() throws Exception { } } - private CloudWatchLogsAsyncClient createCloudWatchAsyncClient(TestMode mode) { - switch (mode) { - case LOCALSTACK: - if (!localStack.isRunning()) { - localStack.start(); - } - return CloudWatchLogsAsyncClient.builder() - .endpointOverride(localStack.getEndpoint()) - .credentialsProvider( - StaticCredentialsProvider.create( - AwsBasicCredentials.create( - localStack.getAccessKey(), localStack.getSecretKey()))) - .region(Region.of(localStack.getRegion())) - .build(); - case MOCKED: - CloudWatchLogsAsyncClient mockClient = Mockito.mock(CloudWatchLogsAsyncClient.class); - - // Mock the responses for log group and stream creation - when(mockClient.createLogGroup(any(CreateLogGroupRequest.class))) - .thenReturn(CompletableFuture.completedFuture(null)); - when(mockClient.createLogStream(any(CreateLogStreamRequest.class))) - .thenReturn(CompletableFuture.completedFuture(null)); - - // Mock the describe log streams response for getting sequence token - DescribeLogStreamsResponse mockStreamsResponse = - DescribeLogStreamsResponse.builder() - .logStreams( - software.amazon.awssdk.services.cloudwatchlogs.model.LogStream.builder() - .logStreamName(LOG_STREAM) - .uploadSequenceToken(null) - .build()) - .build(); - when(mockClient.describeLogStreams(any(DescribeLogStreamsRequest.class))) - .thenReturn(CompletableFuture.completedFuture(mockStreamsResponse)); - - // Mock the putLogEvents response - PutLogEventsResponse mockPutResponse = - PutLogEventsResponse.builder().nextSequenceToken("mock-sequence-token-123").build(); - when(mockClient.putLogEvents(any(PutLogEventsRequest.class))) - .thenReturn(CompletableFuture.completedFuture(mockPutResponse)); - - return mockClient; - default: - throw new IllegalArgumentException("Unknown test mode: " + mode); + private CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { + if (!localStack.isRunning()) { + localStack.start(); } + return CloudWatchLogsAsyncClient.builder() + .endpointOverride(localStack.getEndpoint()) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(localStack.getAccessKey(), localStack.getSecretKey()))) + .region(Region.of(localStack.getRegion())) + .build(); } private AwsCloudWatchEventListener createListener(CloudWatchLogsAsyncClient client) { @@ -199,64 +143,47 @@ protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { return listener; } - @ParameterizedTest - @MethodSource("testModeProvider") - void shouldCreateLogGroupAndStream(TestMode mode) { - CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(mode); + @Test + void shouldCreateLogGroupAndStream() { + CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(); AwsCloudWatchEventListener listener = createListener(client); // Start the listener which should create the log group and stream listener.start(); - if (mode == TestMode.LOCALSTACK) { - // Verify log group exists - DescribeLogGroupsResponse groups = - client - .describeLogGroups( - DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) - .join(); - assertThat(groups.logGroups()) - .hasSize(1) - .first() - .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); - - // Verify log stream exists - DescribeLogStreamsResponse streams = - client - .describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamNamePrefix(LOG_STREAM) - .build()) - .join(); - assertThat(streams.logStreams()) - .hasSize(1) - .first() - .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); - } else { - // Verify method calls for mocked client - verify(client, times(1)) - .createLogGroup( - argThat((CreateLogGroupRequest request) -> request.logGroupName().equals(LOG_GROUP))); - verify(client, times(1)) - .createLogStream( - argThat( - (CreateLogStreamRequest request) -> - request.logGroupName().equals(LOG_GROUP) - && request.logStreamName().equals(LOG_STREAM))); - } + // Verify log group exists + DescribeLogGroupsResponse groups = + client + .describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) + .join(); + assertThat(groups.logGroups()) + .hasSize(1) + .first() + .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); + + // Verify log stream exists + DescribeLogStreamsResponse streams = + client + .describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamNamePrefix(LOG_STREAM) + .build()) + .join(); + assertThat(streams.logStreams()) + .hasSize(1) + .first() + .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); // Clean up listener.shutdown(); - if (client != null && mode == TestMode.LOCALSTACK) { - client.close(); - } + client.close(); } - @ParameterizedTest - @MethodSource("testModeProvider") - void shouldSendEventToCloudWatch(TestMode mode) { - CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(mode); + @Test + void shouldSendEventToCloudWatch() { + CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(); AwsCloudWatchEventListener listener = createListener(client); // Start the listener @@ -267,31 +194,19 @@ void shouldSendEventToCloudWatch(TestMode mode) { AfterTableRefreshedEvent event = new AfterTableRefreshedEvent(testTable); listener.onAfterTableRefreshed(event); - if (mode == TestMode.LOCALSTACK) { - // Verify the event was sent to CloudWatch by reading the actual logs - GetLogEventsResponse logEvents; - int eventCount = 0; - long startTime = System.currentTimeMillis(); - while (eventCount == 0) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - fail("Thread interrupted waiting for event"); - } - if (System.currentTimeMillis() - startTime > 30_000) { - fail("Timeout exceeded while waiting for event to arrive"); - } - logEvents = - client - .getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()) - .join(); - eventCount = logEvents.events().size(); + // Verify the event was sent to CloudWatch by reading the actual logs + GetLogEventsResponse logEvents; + int eventCount = 0; + long startTime = System.currentTimeMillis(); + while (eventCount == 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + fail("Thread interrupted waiting for event"); + } + if (System.currentTimeMillis() - startTime > 30_000) { + fail("Timeout exceeded while waiting for event to arrive"); } - logEvents = client .getLogEvents( @@ -300,51 +215,38 @@ void shouldSendEventToCloudWatch(TestMode mode) { .logStreamName(LOG_STREAM) .build()) .join(); - - assertThat(logEvents.events()) - .hasSize(1) - .first() - .satisfies( - logEvent -> { - String message = logEvent.message(); - assertThat(message).contains(REALM); - assertThat(message).contains(AfterTableRefreshedEvent.class.getSimpleName()); - assertThat(message).contains(TEST_USER); - assertThat(message).contains(testTable.toString()); - }); - } else { - // Verify that putLogEvents was called with the expected content - verify(client, times(1)) - .putLogEvents( - argThat( - (PutLogEventsRequest request) -> { - // Verify basic request structure - assertThat(request.logGroupName()).isEqualTo(LOG_GROUP); - assertThat(request.logStreamName()).isEqualTo(LOG_STREAM); - assertThat(request.logEvents()).hasSize(1); - - // Verify the log event content - String logMessage = request.logEvents().getFirst().message(); - assertThat(logMessage).contains(REALM); - assertThat(logMessage).contains(TEST_USER); - assertThat(logMessage).contains("AfterTableRefreshedEvent"); - assertThat(logMessage).contains(testTable.toString()); - - return true; - })); + eventCount = logEvents.events().size(); } + logEvents = + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); + + assertThat(logEvents.events()) + .hasSize(1) + .first() + .satisfies( + logEvent -> { + String message = logEvent.message(); + assertThat(message).contains(REALM); + assertThat(message).contains(AfterTableRefreshedEvent.class.getSimpleName()); + assertThat(message).contains(TEST_USER); + assertThat(message).contains(testTable.toString()); + }); + // Clean up listener.shutdown(); - if (client != null && mode == TestMode.LOCALSTACK) { - client.close(); - } + client.close(); } - @ParameterizedTest - @MethodSource("testModeProvider") - void handleSynchronousModeCorrectly(TestMode mode) { - CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(mode); + @Test + void handleSynchronousModeCorrectly() { + CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(); // Test synchronous mode when(config.synchronousMode()).thenReturn(true); @@ -356,31 +258,19 @@ void handleSynchronousModeCorrectly(TestMode mode) { AfterTableRefreshedEvent syncEvent = new AfterTableRefreshedEvent(syncTestTable); syncListener.onAfterTableRefreshed(syncEvent); - if (mode == TestMode.LOCALSTACK) { - // Verify both events were sent to CloudWatch - GetLogEventsResponse logEvents; - int eventCount = 0; - long startTime = System.currentTimeMillis(); - while (eventCount == 0) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - fail("Thread interrupted waiting for event"); - } - if (System.currentTimeMillis() - startTime > 30_000) { - fail("Timeout exceeded while waiting for event to arrive"); - } - logEvents = - client - .getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()) - .join(); - eventCount = logEvents.events().size(); + // Verify both events were sent to CloudWatch + GetLogEventsResponse logEvents; + int eventCount = 0; + long startTime = System.currentTimeMillis(); + while (eventCount == 0) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + fail("Thread interrupted waiting for event"); + } + if (System.currentTimeMillis() - startTime > 30_000) { + fail("Timeout exceeded while waiting for event to arrive"); } - logEvents = client .getLogEvents( @@ -389,43 +279,32 @@ void handleSynchronousModeCorrectly(TestMode mode) { .logStreamName(LOG_STREAM) .build()) .join(); - - assertThat(logEvents.events()).hasSize(1); - - // Verify sync event - assertThat(logEvents.events()) - .anySatisfy( - logEvent -> { - String message = logEvent.message(); - assertThat(message).contains("test_table_sync"); - assertThat(message).contains("AfterTableRefreshedEvent"); - }); - } else { - // Verify that putLogEvents was called with the expected content - verify(client, times(1)) - .putLogEvents( - argThat( - (PutLogEventsRequest request) -> { - // Verify basic request structure - assertThat(request.logGroupName()).isEqualTo(LOG_GROUP); - assertThat(request.logStreamName()).isEqualTo(LOG_STREAM); - assertThat(request.logEvents()).hasSize(1); - - // Verify the log event content - String logMessage = request.logEvents().getFirst().message(); - assertThat(logMessage).contains(REALM); - assertThat(logMessage).contains(TEST_USER); - assertThat(logMessage).contains("AfterTableRefreshedEvent"); - assertThat(logMessage).contains("test_table_sync"); - - return true; - })); + eventCount = logEvents.events().size(); } + logEvents = + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); + + assertThat(logEvents.events()).hasSize(1); + + // Verify sync event + assertThat(logEvents.events()) + .anySatisfy( + logEvent -> { + String message = logEvent.message(); + assertThat(message).contains("test_table_sync"); + assertThat(message).contains("AfterTableRefreshedEvent"); + }); + // Clean up syncListener.shutdown(); - if (client != null && mode == TestMode.LOCALSTACK) { - client.close(); - } + + client.close(); } } From b0c616098c2035e0426ec43be1f36891a0ed35d6 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Fri, 22 Aug 2025 01:07:57 -0700 Subject: [PATCH 31/39] typo --- .../AwsCloudWatchEventListenerTest.java | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index fa8ec2f9a5..f0b6d88d45 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -151,34 +151,36 @@ void shouldCreateLogGroupAndStream() { // Start the listener which should create the log group and stream listener.start(); - // Verify log group exists - DescribeLogGroupsResponse groups = - client - .describeLogGroups( - DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) - .join(); - assertThat(groups.logGroups()) - .hasSize(1) - .first() - .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); - - // Verify log stream exists - DescribeLogStreamsResponse streams = - client - .describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamNamePrefix(LOG_STREAM) - .build()) - .join(); - assertThat(streams.logStreams()) - .hasSize(1) - .first() - .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + try { + // Verify log group exists + DescribeLogGroupsResponse groups = + client + .describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) + .join(); + assertThat(groups.logGroups()) + .hasSize(1) + .first() + .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); - // Clean up - listener.shutdown(); - client.close(); + // Verify log stream exists + DescribeLogStreamsResponse streams = + client + .describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamNamePrefix(LOG_STREAM) + .build()) + .join(); + assertThat(streams.logStreams()) + .hasSize(1) + .first() + .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + } finally { + // Clean up + listener.shutdown(); + client.close(); + } } @Test From 44fad9a5cafb82a19666e1d4696ee02bd792335f Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Fri, 22 Aug 2025 13:14:33 -0700 Subject: [PATCH 32/39] spotlessapply --- .../AwsCloudWatchEventListenerTest.java | 121 +++++++++--------- 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index f0b6d88d45..08f132211b 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -154,28 +154,28 @@ void shouldCreateLogGroupAndStream() { try { // Verify log group exists DescribeLogGroupsResponse groups = - client - .describeLogGroups( - DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) - .join(); + client + .describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) + .join(); assertThat(groups.logGroups()) - .hasSize(1) - .first() - .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); + .hasSize(1) + .first() + .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); // Verify log stream exists DescribeLogStreamsResponse streams = - client - .describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamNamePrefix(LOG_STREAM) - .build()) - .join(); + client + .describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamNamePrefix(LOG_STREAM) + .build()) + .join(); assertThat(streams.logStreams()) - .hasSize(1) - .first() - .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + .hasSize(1) + .first() + .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); } finally { // Clean up listener.shutdown(); @@ -208,35 +208,36 @@ void shouldSendEventToCloudWatch() { fail("Timeout exceeded while waiting for event to arrive"); } logEvents = - client.getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()) - .join(); + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); eventCount = logEvents.events().size(); } logEvents = - client - .getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()) - .join(); + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); assertThat(logEvents.events()) - .hasSize(1) - .first() - .satisfies( - logEvent -> { - String message = logEvent.message(); - assertThat(message).contains(REALM); - assertThat(message).contains(AfterTableRefreshedEvent.class.getSimpleName()); - assertThat(message).contains(TEST_USER); - assertThat(message).contains(testTable.toString()); - }); + .hasSize(1) + .first() + .satisfies( + logEvent -> { + String message = logEvent.message(); + assertThat(message).contains(REALM); + assertThat(message).contains(AfterTableRefreshedEvent.class.getSimpleName()); + assertThat(message).contains(TEST_USER); + assertThat(message).contains(testTable.toString()); + }); } finally { // Clean up listener.shutdown(); @@ -272,35 +273,35 @@ void shouldSendEventInSynchronousMode() { fail("Timeout exceeded while waiting for event to arrive"); } logEvents = - client - .getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()) - .join(); + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); eventCount = logEvents.events().size(); } logEvents = - client - .getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()) - .join(); + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); assertThat(logEvents.events()).hasSize(1); // Verify sync event assertThat(logEvents.events()) - .anySatisfy( - logEvent -> { - String message = logEvent.message(); - assertThat(message).contains("test_table_sync"); - assertThat(message).contains("AfterTableRefreshedEvent"); - }); + .anySatisfy( + logEvent -> { + String message = logEvent.message(); + assertThat(message).contains("test_table_sync"); + assertThat(message).contains("AfterTableRefreshedEvent"); + }); } finally { // Clean up syncListener.shutdown(); From e8447a94296eeb5f1fa22b0fee34b5c2a058ec78 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 25 Aug 2025 11:52:40 -0700 Subject: [PATCH 33/39] fix docstrings --- .../aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java index eca04d1ecb..d98ac70999 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java @@ -44,7 +44,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur * *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-group} * - * @return an Optional containing the log group name, or empty if not configured + * @return a String containing the log group name, or the default value if not configured */ @WithName("log-group") @WithDefault("polaris-cloudwatch-default-group") @@ -59,7 +59,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur * *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-stream} * - * @return an Optional containing the log stream name, or empty if not configured + * @return a String containing the log stream name, or the default value if not configured */ @WithName("log-stream") @WithDefault("polaris-cloudwatch-default-stream") @@ -74,7 +74,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur * *

Configuration property: {@code polaris.event-listener.aws-cloudwatch.region} * - * @return an Optional containing the AWS region, or empty if not configured + * @return a String containing the AWS region, or the default value if not configured */ @WithName("region") @WithDefault("us-east-1") From 688ec977e23ae66061fddc752e9ceda547a87559 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 25 Aug 2025 12:02:28 -0700 Subject: [PATCH 34/39] refactor --- .../aws/cloudwatch/AwsCloudWatchEventListener.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index 8828ba4809..72068c69a1 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -134,7 +134,8 @@ protected void transformAndSendEvent(HashMap properties) { LOGGER.error("Error processing event into JSON string: ", e); return; } - InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp()); + InputLogEvent inputLogEvent = + InputLogEvent.builder().message(eventAsJson).timestamp(clock.millis()).build(); PutLogEventsRequest.Builder requestBuilder = PutLogEventsRequest.builder() .logGroupName(logGroup) @@ -154,12 +155,4 @@ protected void transformAndSendEvent(HashMap properties) { future.join(); } } - - private long getCurrentTimestamp() { - return clock.millis(); - } - - private InputLogEvent createLogEvent(String eventAsJson, long timestamp) { - return InputLogEvent.builder().message(eventAsJson).timestamp(timestamp).build(); - } } From 360cb9134b6f1a3c7aab757bade55e8a6a3cd08e Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Mon, 25 Aug 2025 13:13:02 -0700 Subject: [PATCH 35/39] use awaitility --- .../AwsCloudWatchEventListenerTest.java | 86 ++++++++----------- 1 file changed, 36 insertions(+), 50 deletions(-) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 08f132211b..65ef4122a4 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -21,12 +21,12 @@ import static org.apache.polaris.containerspec.ContainerSpecHelper.containerSpecHelper; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; import static org.mockito.Mockito.when; import jakarta.ws.rs.core.SecurityContext; import java.security.Principal; import java.time.Clock; +import java.time.Duration; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -35,6 +35,7 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.service.events.AfterTableRefreshedEvent; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -194,31 +195,23 @@ void shouldSendEventToCloudWatch() { AfterTableRefreshedEvent event = new AfterTableRefreshedEvent(testTable); listener.onAfterTableRefreshed(event); - // Verify the event was sent to CloudWatch by reading the actual logs - GetLogEventsResponse logEvents; - int eventCount = 0; - long startTime = System.currentTimeMillis(); - while (eventCount == 0) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - fail("Thread interrupted waiting for event"); - } - if (System.currentTimeMillis() - startTime > 30_000) { - fail("Timeout exceeded while waiting for event to arrive"); - } - logEvents = - client - .getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()) - .join(); - eventCount = logEvents.events().size(); - } - - logEvents = + Awaitility.await("expected amount of records should be sent to CloudWatch") + .atMost(Duration.ofSeconds(30)) + .pollDelay(Duration.ofMillis(100)) + .pollInterval(Duration.ofMillis(100)) + .untilAsserted( + () -> { + GetLogEventsResponse resp = + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); + assertThat(resp.events().size()).isGreaterThan(0); + }); + GetLogEventsResponse logEvents = client .getLogEvents( GetLogEventsRequest.builder() @@ -259,31 +252,24 @@ void shouldSendEventInSynchronousMode() { AfterTableRefreshedEvent syncEvent = new AfterTableRefreshedEvent(syncTestTable); syncListener.onAfterTableRefreshed(syncEvent); - // Verify both events were sent to CloudWatch - GetLogEventsResponse logEvents; - int eventCount = 0; - long startTime = System.currentTimeMillis(); - while (eventCount == 0) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - fail("Thread interrupted waiting for event"); - } - if (System.currentTimeMillis() - startTime > 30_000) { - fail("Timeout exceeded while waiting for event to arrive"); - } - logEvents = - client - .getLogEvents( - GetLogEventsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamName(LOG_STREAM) - .build()) - .join(); - eventCount = logEvents.events().size(); - } + Awaitility.await("expected amount of records should be sent to CloudWatch") + .atMost(Duration.ofSeconds(30)) + .pollDelay(Duration.ofMillis(100)) + .pollInterval(Duration.ofMillis(100)) + .untilAsserted( + () -> { + GetLogEventsResponse resp = + client + .getLogEvents( + GetLogEventsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); + assertThat(resp.events().size()).isGreaterThan(0); + }); - logEvents = + GetLogEventsResponse logEvents = client .getLogEvents( GetLogEventsRequest.builder() From 7c09d9b19f061bd1be4e2d9a7b6a821cb76be029 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Thu, 28 Aug 2025 13:29:07 -0700 Subject: [PATCH 36/39] Add negative case testing --- .../AwsCloudWatchEventListenerTest.java | 72 ++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 65ef4122a4..78ff713c1a 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -49,6 +49,8 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsAsyncClient; +import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; @@ -153,34 +155,27 @@ void shouldCreateLogGroupAndStream() { listener.start(); try { - // Verify log group exists - DescribeLogGroupsResponse groups = - client - .describeLogGroups( - DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) - .join(); - assertThat(groups.logGroups()) - .hasSize(1) - .first() - .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); - - // Verify log stream exists - DescribeLogStreamsResponse streams = - client - .describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamNamePrefix(LOG_STREAM) - .build()) - .join(); - assertThat(streams.logStreams()) - .hasSize(1) - .first() - .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + verifyLogGroupAndStreamExist(client); } finally { - // Clean up + client.close(); listener.shutdown(); + } + } + + @Test + void shouldAcceptPreviouslyCreatedLogGroupAndStream() { + CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(); + client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(LOG_GROUP).build()).join(); + client.createLogStream(CreateLogStreamRequest.builder().logGroupName(LOG_GROUP).logStreamName(LOG_STREAM).build()).join(); + verifyLogGroupAndStreamExist(client); + + AwsCloudWatchEventListener listener = createListener(client); + listener.start(); + try { + verifyLogGroupAndStreamExist(client); + } finally { client.close(); + listener.shutdown(); } } @@ -294,4 +289,31 @@ void shouldSendEventInSynchronousMode() { client.close(); } } + + private void verifyLogGroupAndStreamExist(CloudWatchLogsAsyncClient client) { + // Verify log group exists + DescribeLogGroupsResponse groups = + client + .describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) + .join(); + assertThat(groups.logGroups()) + .hasSize(1) + .first() + .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); + + // Verify log stream exists + DescribeLogStreamsResponse streams = + client + .describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamNamePrefix(LOG_STREAM) + .build()) + .join(); + assertThat(streams.logStreams()) + .hasSize(1) + .first() + .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + } } From 5e348846ae7c4eadb6e9396efd43736e6df0004e Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Thu, 28 Aug 2025 13:29:47 -0700 Subject: [PATCH 37/39] spotlessapply --- .../AwsCloudWatchEventListenerTest.java | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 78ff713c1a..8401493037 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -166,7 +166,13 @@ void shouldCreateLogGroupAndStream() { void shouldAcceptPreviouslyCreatedLogGroupAndStream() { CloudWatchLogsAsyncClient client = createCloudWatchAsyncClient(); client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(LOG_GROUP).build()).join(); - client.createLogStream(CreateLogStreamRequest.builder().logGroupName(LOG_GROUP).logStreamName(LOG_STREAM).build()).join(); + client + .createLogStream( + CreateLogStreamRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamName(LOG_STREAM) + .build()) + .join(); verifyLogGroupAndStreamExist(client); AwsCloudWatchEventListener listener = createListener(client); @@ -293,27 +299,27 @@ void shouldSendEventInSynchronousMode() { private void verifyLogGroupAndStreamExist(CloudWatchLogsAsyncClient client) { // Verify log group exists DescribeLogGroupsResponse groups = - client - .describeLogGroups( - DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) - .join(); + client + .describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(LOG_GROUP).build()) + .join(); assertThat(groups.logGroups()) - .hasSize(1) - .first() - .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); + .hasSize(1) + .first() + .satisfies(group -> assertThat(group.logGroupName()).isEqualTo(LOG_GROUP)); // Verify log stream exists DescribeLogStreamsResponse streams = - client - .describeLogStreams( - DescribeLogStreamsRequest.builder() - .logGroupName(LOG_GROUP) - .logStreamNamePrefix(LOG_STREAM) - .build()) - .join(); + client + .describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(LOG_GROUP) + .logStreamNamePrefix(LOG_STREAM) + .build()) + .join(); assertThat(streams.logStreams()) - .hasSize(1) - .first() - .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); + .hasSize(1) + .first() + .satisfies(stream -> assertThat(stream.logStreamName()).isEqualTo(LOG_STREAM)); } } From e2ed7439d4c64d72cfdb6b4a0a5135f5db1a2708 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Sun, 31 Aug 2025 18:58:48 -0700 Subject: [PATCH 38/39] Revision based on comments from @eric-maynard and @singhpk234 --- ...ner.java => PropertyMapEventListener.java} | 13 ++- .../AwsCloudWatchConfiguration.java | 6 +- .../AwsCloudWatchEventListener.java | 101 +++++++++++------- .../QuarkusAwsCloudWatchConfiguration.java | 6 +- .../AwsCloudWatchEventListenerTest.java | 26 ++++- .../in-dev/unreleased/configuration.md | 90 ++++++++-------- 6 files changed, 143 insertions(+), 99 deletions(-) rename runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/{JsonEventListener.java => PropertyMapEventListener.java} (74%) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/PropertyMapEventListener.java similarity index 74% rename from runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java rename to runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/PropertyMapEventListener.java index 4e4ab65e3c..28db713dbd 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/PropertyMapEventListener.java @@ -24,14 +24,13 @@ import org.apache.polaris.service.events.PolarisEventListener; /** - * Abstract base class from which all event sinks that output events in JSON format can extend. - * - *

This class provides a common framework for transforming Polaris events into JSON format and - * sending them to various destinations. Concrete implementations should override the {@link - * #transformAndSendEvent(HashMap)} method to define how the JSON event data should be transmitted - * or stored. + * This class provides a common framework for transforming Polaris events into a HashMap, which can + * be used to transform the event further, such as transforming into a JSON string, and send them to + * various destinations. Concrete implementations should override the + * {{@code @link#transformAndSendEvent(HashMap)}} method to define how the event data should be + * transformed into a JSON string, transmitted, and/or stored. */ -public abstract class JsonEventListener extends PolarisEventListener { +public abstract class PropertyMapEventListener extends PolarisEventListener { protected abstract void transformAndSendEvent(HashMap properties); @Override diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java index 64f55fba8f..46e07f569d 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java @@ -21,11 +21,11 @@ /** Configuration interface for AWS CloudWatch event listener settings. */ public interface AwsCloudWatchConfiguration { - String awsCloudwatchlogGroup(); + String awsCloudWatchLogGroup(); - String awsCloudwatchlogStream(); + String awsCloudWatchLogStream(); - String awsCloudwatchRegion(); + String awsCloudWatchRegion(); boolean synchronousMode(); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index 72068c69a1..3e16eed99b 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -32,27 +32,27 @@ import java.util.HashMap; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; +import java.util.function.Supplier; import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.events.jsonEventListener.JsonEventListener; +import org.apache.polaris.service.config.PolarisIcebergObjectMapperCustomizer; +import org.apache.polaris.service.events.jsonEventListener.PropertyMapEventListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsAsyncClient; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest; -import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupResponse; import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest; -import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamResponse; +import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsRequest; +import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogStreamsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.InputLogEvent; import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsRequest; import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsResponse; -import software.amazon.awssdk.services.cloudwatchlogs.model.ResourceAlreadyExistsException; @ApplicationScoped @Identifier("aws-cloudwatch") -public class AwsCloudWatchEventListener extends JsonEventListener { +public class AwsCloudWatchEventListener extends PropertyMapEventListener { private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListener.class); - private final ObjectMapper objectMapper = new ObjectMapper(); + final ObjectMapper objectMapper = new ObjectMapper(); private CloudWatchLogsAsyncClient client; @@ -67,12 +67,16 @@ public class AwsCloudWatchEventListener extends JsonEventListener { @Context SecurityContext securityContext; @Inject - public AwsCloudWatchEventListener(AwsCloudWatchConfiguration config, Clock clock) { - this.logStream = config.awsCloudwatchlogStream(); - this.logGroup = config.awsCloudwatchlogGroup(); - this.region = Region.of(config.awsCloudwatchRegion()); + public AwsCloudWatchEventListener( + AwsCloudWatchConfiguration config, + Clock clock, + PolarisIcebergObjectMapperCustomizer customizer) { + this.logStream = config.awsCloudWatchLogStream(); + this.logGroup = config.awsCloudWatchLogGroup(); + this.region = Region.of(config.awsCloudWatchRegion()); this.synchronousMode = config.synchronousMode(); this.clock = clock; + customizer.customize(this.objectMapper); } @PostConstruct @@ -86,32 +90,55 @@ protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { } private void ensureLogGroupAndStream() { - try { - CompletableFuture future = - client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()); - future.join(); - } catch (CompletionException e) { - if (e.getCause() instanceof ResourceAlreadyExistsException) { - LOGGER.debug("Log group {} already exists", logGroup); - } else { - throw e; - } - } + ensureResourceExists( + () -> + client + .describeLogGroups( + DescribeLogGroupsRequest.builder().logGroupNamePrefix(logGroup).build()) + .join() + .logGroups() + .stream() + .anyMatch(g -> g.logGroupName().equals(logGroup)), + () -> + client + .createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build()) + .join(), + "group", + logGroup); + ensureResourceExists( + () -> + client + .describeLogStreams( + DescribeLogStreamsRequest.builder() + .logGroupName(logGroup) + .logStreamNamePrefix(logStream) + .build()) + .join() + .logStreams() + .stream() + .anyMatch(s -> s.logStreamName().equals(logStream)), + () -> + client + .createLogStream( + CreateLogStreamRequest.builder() + .logGroupName(logGroup) + .logStreamName(logStream) + .build()) + .join(), + "stream", + logStream); + } - try { - CompletableFuture future = - client.createLogStream( - CreateLogStreamRequest.builder() - .logGroupName(logGroup) - .logStreamName(logStream) - .build()); - future.join(); - } catch (CompletionException e) { - if (e.getCause() instanceof ResourceAlreadyExistsException) { - LOGGER.debug("Log stream {} already exists", logStream); - } else { - throw e; - } + private static void ensureResourceExists( + Supplier existsCheck, + Runnable createAction, + String resourceType, + String resourceName) { + if (existsCheck.get()) { + LOGGER.debug("Log {} [{}] already exists", resourceType, resourceName); + } else { + LOGGER.debug("Attempting to create log {}: {}", resourceType, resourceName); + createAction.run(); } } @@ -124,7 +151,7 @@ void shutdown() { @Override protected void transformAndSendEvent(HashMap properties) { - properties.put("realm", callContext.getRealmContext().getRealmIdentifier()); + properties.put("realm_id", callContext.getRealmContext().getRealmIdentifier()); properties.put("principal", securityContext.getUserPrincipal().getName()); // TODO: Add request ID when it is available String eventAsJson; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java index d98ac70999..91c32ce063 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java @@ -49,7 +49,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur @WithName("log-group") @WithDefault("polaris-cloudwatch-default-group") @Override - String awsCloudwatchlogGroup(); + String awsCloudWatchLogGroup(); /** * Returns the AWS CloudWatch log stream name for event logging. @@ -64,7 +64,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur @WithName("log-stream") @WithDefault("polaris-cloudwatch-default-stream") @Override - String awsCloudwatchlogStream(); + String awsCloudWatchLogStream(); /** * Returns the AWS region where CloudWatch logs should be sent. @@ -79,7 +79,7 @@ public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfigur @WithName("region") @WithDefault("us-east-1") @Override - String awsCloudwatchRegion(); + String awsCloudWatchRegion(); /** * Returns the synchronous mode setting for CloudWatch logging. diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index 8401493037..f3cac32196 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -23,7 +23,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import io.quarkus.runtime.configuration.MemorySize; import jakarta.ws.rs.core.SecurityContext; +import java.math.BigInteger; import java.security.Principal; import java.time.Clock; import java.time.Duration; @@ -34,6 +37,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.service.config.PolarisIcebergObjectMapperCustomizer; import org.apache.polaris.service.events.AfterTableRefreshedEvent; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; @@ -73,6 +77,9 @@ class AwsCloudWatchEventListenerTest { private static final String REALM = "test-realm"; private static final String TEST_USER = "test-user"; private static final Clock clock = Clock.systemUTC(); + private static final BigInteger MAX_BODY_SIZE = BigInteger.valueOf(1024 * 1024); + private static final PolarisIcebergObjectMapperCustomizer customizer = + new PolarisIcebergObjectMapperCustomizer(new MemorySize(MAX_BODY_SIZE)); @Mock private AwsCloudWatchConfiguration config; @@ -85,9 +92,9 @@ void setUp() { executorService = Executors.newSingleThreadExecutor(); // Configure the mocks - when(config.awsCloudwatchlogGroup()).thenReturn(LOG_GROUP); - when(config.awsCloudwatchlogStream()).thenReturn(LOG_STREAM); - when(config.awsCloudwatchRegion()).thenReturn("us-east-1"); + when(config.awsCloudWatchLogGroup()).thenReturn(LOG_GROUP); + when(config.awsCloudWatchLogStream()).thenReturn(LOG_STREAM); + when(config.awsCloudWatchRegion()).thenReturn("us-east-1"); when(config.synchronousMode()).thenReturn(false); // Default to async mode } @@ -122,7 +129,7 @@ private CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { private AwsCloudWatchEventListener createListener(CloudWatchLogsAsyncClient client) { AwsCloudWatchEventListener listener = - new AwsCloudWatchEventListener(config, clock) { + new AwsCloudWatchEventListener(config, clock, customizer) { @Override protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { return client; @@ -296,6 +303,17 @@ void shouldSendEventInSynchronousMode() { } } + @Test + void ensureObjectMapperCustomizerIsApplied() { + AwsCloudWatchEventListener listener = createListener(createCloudWatchAsyncClient()); + listener.start(); + + assertThat(listener.objectMapper.getPropertyNamingStrategy()) + .isInstanceOf(PropertyNamingStrategies.KebabCaseStrategy.class); + assertThat(listener.objectMapper.getFactory().streamReadConstraints().getMaxDocumentLength()) + .isEqualTo(MAX_BODY_SIZE.longValue()); + } + private void verifyLogGroupAndStreamExist(CloudWatchLogsAsyncClient client) { // Verify log group exists DescribeLogGroupsResponse groups = diff --git a/site/content/in-dev/unreleased/configuration.md b/site/content/in-dev/unreleased/configuration.md index 5e0ba3f4f6..103b8c9486 100644 --- a/site/content/in-dev/unreleased/configuration.md +++ b/site/content/in-dev/unreleased/configuration.md @@ -78,51 +78,51 @@ read-only mode, as Polaris only reads the configuration file once, at startup. ## Polaris Configuration Options Reference -| Configuration Property | Default Value | Description | -|----------------------------------------------------------------------------------------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `polaris.persistence.type` | `relational-jdbc` | Define the persistence backend used by Polaris (`in-memory`, `relational-jdbc`, `eclipse-link` (deprecated)). See [Configuring Apache Polaris for Production)[{{% ref "configuring-polaris-for-production.md" %}}) | -| `polaris.persistence.relational.jdbc.max-retries` | `1` | Total number of retries JDBC persistence will attempt on connection resets or serialization failures before giving up. | -| `polaris.persistence.relational.jdbc.max_duaration_in_ms` | `5000 ms` | Max time interval (ms) since the start of a transaction when retries can be attempted. | -| `polaris.persistence.relational.jdbc.initial_delay_in_ms` | `100 ms` | Initial delay before retrying. The delay is doubled after each retry. | -| `polaris.persistence.eclipselink.configurationFile` | | Define the location of the `persistence.xml`. By default, it's the built-in `persistence.xml` in use. | -| `polaris.persistence.eclipselink.persistenceUnit` | `polaris` | Define the name of the persistence unit to use, as defined in the `persistence.xml`. | -| `polaris.realm-context.type` | `default` | Define the type of the Polaris realm to use. | -| `polaris.realm-context.realms` | `POLARIS` | Define the list of realms to use. | -| `polaris.realm-context.header-name` | `Polaris-Realm` | Define the header name defining the realm context. | -| `polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"` | `false` | Flag to enforce check if credential rotation. | -| `polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"` | `FILE` | Define the catalog supported storage. Supported values are `S3`, `GCS`, `AZURE`, `FILE`. | -| `polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"` | `true` | "Override" realm features, here the skip credential subscoping indirection flag. | -| `polaris.authentication.authenticator.type` | `default` | Define the Polaris authenticator type. | -| `polaris.authentication.token-service.type` | `default` | Define the Polaris token service type. | -| `polaris.authentication.token-broker.type` | `rsa-key-pair` | Define the Polaris token broker type. Also configure the location of the key files. For RSA: if the locations of the key files are not configured, an ephemeral key-pair will be created on each Polaris server instance startup, which breaks existing tokens after server restarts and is also incompatible with running multiple Polaris server instances. | -| `polaris.authentication.token-broker.max-token-generation` | `PT1H` | Define the max token generation policy on the token broker. | -| `polaris.authentication.token-broker.rsa-key-pair.private-key-file` | | Define the location of the RSA-256 private key file, if present the `public-key` file must be specified, too. | -| `polaris.authentication.token-broker.rsa-key-pair.public-key-file` | | Define the location of the RSA-256 public key file, if present the `private-key` file must be specified, too. | -| `polaris.authentication.token-broker.symmetric-key.secret` | `secret` | Define the secret of the symmetric key. | -| `polaris.authentication.token-broker.symmetric-key.file` | `/tmp/symmetric.key` | Define the location of the symmetric key file. | -| `polaris.storage.aws.access-key` | `accessKey` | Define the AWS S3 access key. If unset, the default credential provider chain will be used. | -| `polaris.storage.aws.secret-key` | `secretKey` | Define the AWS S3 secret key. If unset, the default credential provider chain will be used. | -| `polaris.storage.gcp.token` | `token` | Define the Google Cloud Storage token. If unset, the default credential provider chain will be used. | -| `polaris.storage.gcp.lifespan` | `PT1H` | Define the Google Cloud Storage lifespan type. If unset, the default credential provider chain will be used. | -| `polaris.log.request-id-header-name` | `Polaris-Request-Id` | Define the header name to match request ID in the log. | -| `polaris.log.mdc.aid` | `polaris` | Define the log context (e.g. MDC) AID. | -| `polaris.log.mdc.sid` | `polaris-service` | Define the log context (e.g. MDC) SID. | -| `polaris.rate-limiter.filter.type` | `no-op` | Define the Polaris rate limiter. Supported values are `no-op`, `token-bucket`. | -| `polaris.rate-limiter.token-bucket.type` | `default` | Define the token bucket rate limiter. | -| `polaris.rate-limiter.token-bucket.requests-per-second` | `9999` | Define the number of requests per second for the token bucket rate limiter. | -| `polaris.rate-limiter.token-bucket.window` | `PT10S` | Define the window type for the token bucket rate limiter. | -| `polaris.metrics.tags.=` | `application=Polaris` | Define arbitrary metric tags to include in every request. | -| `polaris.metrics.realm-id-tag.api-metrics-enabled` | `false` | Whether to enable the `realm_id` metric tag in API metrics. | -| `polaris.metrics.realm-id-tag.http-metrics-enabled` | `false` | Whether to enable the `realm_id` metric tag in HTTP request metrics. | -| `polaris.metrics.realm-id-tag.http-metrics-max-cardinality` | `100` | The maximum cardinality for the `realm_id` tag in HTTP request metrics. | -| `polaris.tasks.max-concurrent-tasks` | `100` | Define the max number of concurrent tasks. | -| `polaris.tasks.max-queued-tasks` | `1000` | Define the max number of tasks in queue. | - | `polaris.config.rollback.compaction.on-conflicts.enabled` | `false` | When set to true Polaris will apply the deconfliction by rollbacking those REPLACE operations snapshots which have the property of `polaris.internal.rollback.compaction.on-conflict` in their snapshot summary set to `rollback`, to resolve conflicts at the server end. | -| `polaris.event-listener.type` | `no-op` | Define the Polaris event listener type. Supported values are `no-op`, `aws-cloudwatch`. | -| `polaris.event-listener.aws-cloudwatch.log-group` | | Define the AWS CloudWatch log group name for the event listener. | -| `polaris.event-listener.aws-cloudwatch.log-stream` | | Define the AWS CloudWatch log stream name for the event listener. | -| `polaris.event-listener.aws-cloudwatch.region` | | Define the AWS region for the CloudWatch event listener. | -| `polaris.event-listener.aws-cloudwatch.synchronous-mode` | `false` | Define whether log events are sent to CloudWatch synchronously. When set to true, events are sent synchronously which may impact performance but ensures immediate delivery. When false (default), events are sent asynchronously for better performance. | +| Configuration Property | Default Value | Description | +|----------------------------------------------------------------------------------------|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `polaris.persistence.type` | `relational-jdbc` | Define the persistence backend used by Polaris (`in-memory`, `relational-jdbc`, `eclipse-link` (deprecated)). See [Configuring Apache Polaris for Production)[{{% ref "configuring-polaris-for-production.md" %}}) | +| `polaris.persistence.relational.jdbc.max-retries` | `1` | Total number of retries JDBC persistence will attempt on connection resets or serialization failures before giving up. | +| `polaris.persistence.relational.jdbc.max_duaration_in_ms` | `5000 ms` | Max time interval (ms) since the start of a transaction when retries can be attempted. | +| `polaris.persistence.relational.jdbc.initial_delay_in_ms` | `100 ms` | Initial delay before retrying. The delay is doubled after each retry. | +| `polaris.persistence.eclipselink.configurationFile` | | Define the location of the `persistence.xml`. By default, it's the built-in `persistence.xml` in use. | +| `polaris.persistence.eclipselink.persistenceUnit` | `polaris` | Define the name of the persistence unit to use, as defined in the `persistence.xml`. | +| `polaris.realm-context.type` | `default` | Define the type of the Polaris realm to use. | +| `polaris.realm-context.realms` | `POLARIS` | Define the list of realms to use. | +| `polaris.realm-context.header-name` | `Polaris-Realm` | Define the header name defining the realm context. | +| `polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"` | `false` | Flag to enforce check if credential rotation. | +| `polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"` | `FILE` | Define the catalog supported storage. Supported values are `S3`, `GCS`, `AZURE`, `FILE`. | +| `polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"` | `true` | "Override" realm features, here the skip credential subscoping indirection flag. | +| `polaris.authentication.authenticator.type` | `default` | Define the Polaris authenticator type. | +| `polaris.authentication.token-service.type` | `default` | Define the Polaris token service type. | +| `polaris.authentication.token-broker.type` | `rsa-key-pair` | Define the Polaris token broker type. Also configure the location of the key files. For RSA: if the locations of the key files are not configured, an ephemeral key-pair will be created on each Polaris server instance startup, which breaks existing tokens after server restarts and is also incompatible with running multiple Polaris server instances. | +| `polaris.authentication.token-broker.max-token-generation` | `PT1H` | Define the max token generation policy on the token broker. | +| `polaris.authentication.token-broker.rsa-key-pair.private-key-file` | | Define the location of the RSA-256 private key file, if present the `public-key` file must be specified, too. | +| `polaris.authentication.token-broker.rsa-key-pair.public-key-file` | | Define the location of the RSA-256 public key file, if present the `private-key` file must be specified, too. | +| `polaris.authentication.token-broker.symmetric-key.secret` | `secret` | Define the secret of the symmetric key. | +| `polaris.authentication.token-broker.symmetric-key.file` | `/tmp/symmetric.key` | Define the location of the symmetric key file. | +| `polaris.storage.aws.access-key` | `accessKey` | Define the AWS S3 access key. If unset, the default credential provider chain will be used. | +| `polaris.storage.aws.secret-key` | `secretKey` | Define the AWS S3 secret key. If unset, the default credential provider chain will be used. | +| `polaris.storage.gcp.token` | `token` | Define the Google Cloud Storage token. If unset, the default credential provider chain will be used. | +| `polaris.storage.gcp.lifespan` | `PT1H` | Define the Google Cloud Storage lifespan type. If unset, the default credential provider chain will be used. | +| `polaris.log.request-id-header-name` | `Polaris-Request-Id` | Define the header name to match request ID in the log. | +| `polaris.log.mdc.aid` | `polaris` | Define the log context (e.g. MDC) AID. | +| `polaris.log.mdc.sid` | `polaris-service` | Define the log context (e.g. MDC) SID. | +| `polaris.rate-limiter.filter.type` | `no-op` | Define the Polaris rate limiter. Supported values are `no-op`, `token-bucket`. | +| `polaris.rate-limiter.token-bucket.type` | `default` | Define the token bucket rate limiter. | +| `polaris.rate-limiter.token-bucket.requests-per-second` | `9999` | Define the number of requests per second for the token bucket rate limiter. | +| `polaris.rate-limiter.token-bucket.window` | `PT10S` | Define the window type for the token bucket rate limiter. | +| `polaris.metrics.tags.=` | `application=Polaris` | Define arbitrary metric tags to include in every request. | +| `polaris.metrics.realm-id-tag.api-metrics-enabled` | `false` | Whether to enable the `realm_id` metric tag in API metrics. | +| `polaris.metrics.realm-id-tag.http-metrics-enabled` | `false` | Whether to enable the `realm_id` metric tag in HTTP request metrics. | +| `polaris.metrics.realm-id-tag.http-metrics-max-cardinality` | `100` | The maximum cardinality for the `realm_id` tag in HTTP request metrics. | +| `polaris.tasks.max-concurrent-tasks` | `100` | Define the max number of concurrent tasks. | +| `polaris.tasks.max-queued-tasks` | `1000` | Define the max number of tasks in queue. | +| `polaris.config.rollback.compaction.on-conflicts.enabled` | `false` | When set to true Polaris will apply the deconfliction by rollbacking those REPLACE operations snapshots which have the property of `polaris.internal.rollback.compaction.on-conflict` in their snapshot summary set to `rollback`, to resolve conflicts at the server end. | +| `polaris.event-listener.type` | `no-op` | Define the Polaris event listener type. Supported values are `no-op`, `aws-cloudwatch`. | +| `polaris.event-listener.aws-cloudwatch.log-group` | `polaris-cloudwatch-default-group` | Define the AWS CloudWatch log group name for the event listener. | +| `polaris.event-listener.aws-cloudwatch.log-stream` | `polaris-cloudwatch-default-stream`| Define the AWS CloudWatch log stream name for the event listener. Ensure that Polaris' IAM credentials have the following actions: "PutLogEvents", "DescribeLogStreams", and "DescribeLogGroups" on the specified log stream/group. If the specified log stream/group does not exist, then "CreateLogStream" and "CreateLogGroup" will also be required. | +| `polaris.event-listener.aws-cloudwatch.region` | `us-east-1` | Define the AWS region for the CloudWatch event listener. | +| `polaris.event-listener.aws-cloudwatch.synchronous-mode` | `false` | Define whether log events are sent to CloudWatch synchronously. When set to true, events are sent synchronously which may impact performance but ensures immediate delivery. When false (default), events are sent asynchronously for better performance. | There are non Polaris configuration properties that can be useful: From a870aeacc0f11fbc1921d79301afa8a64d495390 Mon Sep 17 00:00:00 2001 From: adnanhemani Date: Tue, 2 Sep 2025 18:45:15 -0700 Subject: [PATCH 39/39] Addressing comments from @singhpk234 --- .../aws/cloudwatch/AwsCloudWatchEventListener.java | 5 +++++ .../aws/cloudwatch/AwsCloudWatchEventListenerTest.java | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java index 3e16eed99b..87cf70a2f1 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.service.config.PolarisIcebergObjectMapperCustomizer; import org.apache.polaris.service.events.jsonEventListener.PropertyMapEventListener; @@ -146,6 +147,7 @@ private static void ensureResourceExists( void shutdown() { if (client != null) { client.close(); + client = null; } } @@ -153,12 +155,15 @@ void shutdown() { protected void transformAndSendEvent(HashMap properties) { properties.put("realm_id", callContext.getRealmContext().getRealmIdentifier()); properties.put("principal", securityContext.getUserPrincipal().getName()); + properties.put( + "activated_roles", ((PolarisPrincipal) securityContext.getUserPrincipal()).getRoles()); // TODO: Add request ID when it is available String eventAsJson; try { eventAsJson = objectMapper.writeValueAsString(properties); } catch (JsonProcessingException e) { LOGGER.error("Error processing event into JSON string: ", e); + LOGGER.debug("Failed to convert the following object into JSON string: {}", properties); return; } InputLogEvent inputLogEvent = diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java index f3cac32196..e7225e1568 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListenerTest.java @@ -30,11 +30,13 @@ import java.security.Principal; import java.time.Clock; import java.time.Duration; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.auth.PolarisPrincipal; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.service.config.PolarisIcebergObjectMapperCustomizer; @@ -141,12 +143,13 @@ protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() { PolarisCallContext polarisCallContext = Mockito.mock(PolarisCallContext.class); RealmContext realmContext = Mockito.mock(RealmContext.class); SecurityContext securityContext = Mockito.mock(SecurityContext.class); - Principal principal = Mockito.mock(Principal.class); + Principal principal = Mockito.mock(PolarisPrincipal.class); when(callContext.getRealmContext()).thenReturn(realmContext); when(callContext.getPolarisCallContext()).thenReturn(polarisCallContext); when(realmContext.getRealmIdentifier()).thenReturn(REALM); when(securityContext.getUserPrincipal()).thenReturn(principal); when(principal.getName()).thenReturn(TEST_USER); + when(((PolarisPrincipal) principal).getRoles()).thenReturn(Set.of("role1", "role2")); listener.callContext = callContext; listener.securityContext = securityContext;