From c702947b3418311a0a772f047be2e1e0ad84cf24 Mon Sep 17 00:00:00 2001 From: iliax Date: Tue, 4 Apr 2023 15:33:14 +0400 Subject: [PATCH 1/6] ISSUE-3144: Spring boot version bump to 3.0.5, snakeyaml upd --- kafka-ui-api/Dockerfile | 2 +- .../GlobalErrorWebExceptionHandler.java | 2 +- .../ui/service/KafkaConfigSanitizer.java | 65 ++++++++++++++----- .../extractor/CognitoAuthorityExtractor.java | 6 +- .../kafka/ui/AbstractIntegrationTest.java | 4 +- .../ui/service/KafkaConfigSanitizerTest.java | 9 ++- kafka-ui-contract/pom.xml | 10 +++ pom.xml | 20 +----- 8 files changed, 73 insertions(+), 45 deletions(-) diff --git a/kafka-ui-api/Dockerfile b/kafka-ui-api/Dockerfile index fcd29c0f069..270908fdfe1 100644 --- a/kafka-ui-api/Dockerfile +++ b/kafka-ui-api/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk-alpine:17-jre +FROM azul/zulu-openjdk-alpine:17-jre-headless RUN apk add --no-cache gcompat # need to make snappy codec work RUN addgroup -S kafkaui && adduser -S kafkaui -G kafkaui diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/GlobalErrorWebExceptionHandler.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/GlobalErrorWebExceptionHandler.java index 394e2aa730b..8ad83fe47dd 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/GlobalErrorWebExceptionHandler.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/GlobalErrorWebExceptionHandler.java @@ -134,7 +134,7 @@ private Mono render(ResponseStatusException exception, ServerReq .timestamp(currentTimestamp()) .stackTrace(Throwables.getStackTraceAsString(exception)); return ServerResponse - .status(exception.getStatus()) + .status(exception.getStatusCode()) .contentType(MediaType.APPLICATION_JSON) .bodyValue(response); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConfigSanitizer.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConfigSanitizer.java index aa267098226..375afb0fefd 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConfigSanitizer.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConfigSanitizer.java @@ -1,38 +1,58 @@ package com.provectus.kafka.ui.service; +import static java.util.regex.Pattern.CASE_INSENSITIVE; + +import com.google.common.collect.ImmutableList; import java.util.Arrays; -import java.util.HashSet; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.config.SslConfigs; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.actuate.endpoint.Sanitizer; import org.springframework.stereotype.Component; @Component -class KafkaConfigSanitizer extends Sanitizer { - private static final List DEFAULT_PATTERNS_TO_SANITIZE = Arrays.asList( - "basic.auth.user.info", /* For Schema Registry credentials */ - "password", "secret", "token", "key", ".*credentials.*", /* General credential patterns */ - "aws.access.*", "aws.secret.*", "aws.session.*" /* AWS-related credential patterns */ - ); +class KafkaConfigSanitizer { + + private static final String SANITIZED_VALUE = "******"; + + private static final String[] REGEX_PARTS = {"*", "$", "^", "+"}; + + private static final List DEFAULT_PATTERNS_TO_SANITIZE = ImmutableList.builder() + .addAll(kafkaConfigKeysToSanitize()) + .add( + "basic.auth.user.info", /* For Schema Registry credentials */ + "password", "secret", "token", "key", ".*credentials.*", /* General credential patterns */ + "aws.access.*", "aws.secret.*", "aws.session.*" /* AWS-related credential patterns */ + ) + .build(); + + private final List sanitizeKeysPatterns; KafkaConfigSanitizer( @Value("${kafka.config.sanitizer.enabled:true}") boolean enabled, @Value("${kafka.config.sanitizer.patterns:}") List patternsToSanitize ) { - if (!enabled) { - setKeysToSanitize(); - } else { - var keysToSanitize = new HashSet<>( - patternsToSanitize.isEmpty() ? DEFAULT_PATTERNS_TO_SANITIZE : patternsToSanitize); - keysToSanitize.addAll(kafkaConfigKeysToSanitize()); - setKeysToSanitize(keysToSanitize.toArray(new String[] {})); - } + this.sanitizeKeysPatterns = enabled + ? compile(patternsToSanitize.isEmpty() ? DEFAULT_PATTERNS_TO_SANITIZE : patternsToSanitize) + : List.of(); + } + + private static List compile(Collection patternStrings) { + return patternStrings.stream() + .map(p -> isRegex(p) + ? Pattern.compile(p, CASE_INSENSITIVE) + : Pattern.compile(".*" + p + "$", CASE_INSENSITIVE)) + .toList(); + } + + private static boolean isRegex(String str) { + return Arrays.stream(REGEX_PARTS).anyMatch(str::contains); } private static Set kafkaConfigKeysToSanitize() { @@ -45,4 +65,17 @@ private static Set kafkaConfigKeysToSanitize() { .collect(Collectors.toSet()); } + public Object sanitize(String key, Object value) { + if (value == null) { + return null; + } + for (Pattern pattern : sanitizeKeysPatterns) { + if (pattern.matcher(key).matches()) { + return SANITIZED_VALUE; + } + } + return value; + } + + } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java index a10e0829cc9..b148abdbd82 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java @@ -1,6 +1,5 @@ package com.provectus.kafka.ui.service.rbac.extractor; -import com.nimbusds.jose.shaded.json.JSONArray; import com.provectus.kafka.ui.model.rbac.Role; import com.provectus.kafka.ui.model.rbac.provider.Provider; import com.provectus.kafka.ui.service.rbac.AccessControlService; @@ -9,6 +8,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; +import org.json.JSONArray; import org.springframework.security.oauth2.core.user.DefaultOAuth2User; import reactor.core.publisher.Mono; @@ -44,6 +44,7 @@ public Mono> extract(AccessControlService acs, Object value, Map> extract(AccessControlService acs, Object value, Map s.getProvider().equals(Provider.OAUTH_COGNITO)) .filter(s -> s.getType().equals("group")) - .anyMatch(subject -> Stream.of(groups.toArray()) + .anyMatch(subject -> Stream.of(groups) .map(Object::toString) - .distinct() .anyMatch(cognitoGroup -> cognitoGroup.equals(subject.getValue())) )) .map(Role::getName) diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java index a30e45fb3ed..dbdfb67fd59 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/AbstractIntegrationTest.java @@ -16,7 +16,7 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ContextConfiguration; -import org.springframework.util.SocketUtils; +import org.springframework.test.util.TestSocketUtils; import org.testcontainers.containers.KafkaContainer; import org.testcontainers.containers.Network; import org.testcontainers.utility.DockerImageName; @@ -61,7 +61,7 @@ public void initialize(@NotNull ConfigurableApplicationContext context) { System.setProperty("kafka.clusters.0.bootstrapServers", kafka.getBootstrapServers()); // List unavailable hosts to verify failover System.setProperty("kafka.clusters.0.schemaRegistry", String.format("http://localhost:%1$s,http://localhost:%1$s,%2$s", - SocketUtils.findAvailableTcpPort(), schemaRegistry.getUrl())); + TestSocketUtils.findAvailableTcpPort(), schemaRegistry.getUrl())); System.setProperty("kafka.clusters.0.kafkaConnect.0.name", "kafka-connect"); System.setProperty("kafka.clusters.0.kafkaConnect.0.userName", "kafka-connect"); System.setProperty("kafka.clusters.0.kafkaConnect.0.password", "kafka-connect"); diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KafkaConfigSanitizerTest.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KafkaConfigSanitizerTest.java index 232e1d37035..6454cd9f2a4 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KafkaConfigSanitizerTest.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/KafkaConfigSanitizerTest.java @@ -5,13 +5,12 @@ import java.util.Arrays; import java.util.Collections; import org.junit.jupiter.api.Test; -import org.springframework.boot.actuate.endpoint.Sanitizer; class KafkaConfigSanitizerTest { @Test void doNothingIfEnabledPropertySetToFalse() { - final Sanitizer sanitizer = new KafkaConfigSanitizer(false, Collections.emptyList()); + final var sanitizer = new KafkaConfigSanitizer(false, Collections.emptyList()); assertThat(sanitizer.sanitize("password", "secret")).isEqualTo("secret"); assertThat(sanitizer.sanitize("sasl.jaas.config", "secret")).isEqualTo("secret"); assertThat(sanitizer.sanitize("database.password", "secret")).isEqualTo("secret"); @@ -19,7 +18,7 @@ void doNothingIfEnabledPropertySetToFalse() { @Test void obfuscateCredentials() { - final Sanitizer sanitizer = new KafkaConfigSanitizer(true, Collections.emptyList()); + final var sanitizer = new KafkaConfigSanitizer(true, Collections.emptyList()); assertThat(sanitizer.sanitize("sasl.jaas.config", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("consumer.sasl.jaas.config", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("producer.sasl.jaas.config", "secret")).isEqualTo("******"); @@ -37,7 +36,7 @@ void obfuscateCredentials() { @Test void notObfuscateNormalConfigs() { - final Sanitizer sanitizer = new KafkaConfigSanitizer(true, Collections.emptyList()); + final var sanitizer = new KafkaConfigSanitizer(true, Collections.emptyList()); assertThat(sanitizer.sanitize("security.protocol", "SASL_SSL")).isEqualTo("SASL_SSL"); final String[] bootstrapServer = new String[] {"test1:9092", "test2:9092"}; assertThat(sanitizer.sanitize("bootstrap.servers", bootstrapServer)).isEqualTo(bootstrapServer); @@ -45,7 +44,7 @@ void notObfuscateNormalConfigs() { @Test void obfuscateCredentialsWithDefinedPatterns() { - final Sanitizer sanitizer = new KafkaConfigSanitizer(true, Arrays.asList("kafka.ui", ".*test.*")); + final var sanitizer = new KafkaConfigSanitizer(true, Arrays.asList("kafka.ui", ".*test.*")); assertThat(sanitizer.sanitize("consumer.kafka.ui", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("this.is.test.credentials", "secret")).isEqualTo("******"); assertThat(sanitizer.sanitize("this.is.not.credential", "not.credential")) diff --git a/kafka-ui-contract/pom.xml b/kafka-ui-contract/pom.xml index 5f6cb58b7d7..b585833875e 100644 --- a/kafka-ui-contract/pom.xml +++ b/kafka-ui-contract/pom.xml @@ -42,6 +42,16 @@ 3.0.2 provided + + javax.annotation + javax.annotation-api + 1.3.2 + + + javax.validation + validation-api + 2.0.1.Final + diff --git a/pom.xml b/pom.xml index 3281ff183ec..5a3f759f36d 100644 --- a/pom.xml +++ b/pom.xml @@ -32,15 +32,13 @@ 2.14.0 0.2.4 3.3.1 - 4.1.85.Final 1.4.2.Final 1.18.24 3.21.9 - 1.1.0 2.13.9 - 1.33 - 2.7.5 - 5.7.5 + 2.0 + 3.0.5 + 6.0.2 1.0.0 0.1.15 0.1.23 @@ -118,13 +116,6 @@ pom import - - io.netty - netty-bom - ${netty.version} - pom - import - com.fasterxml.jackson jackson-bom @@ -147,11 +138,6 @@ protobuf-java ${protobuf-java.version} - - io.projectreactor.netty - reactor-netty-http - ${reactor-netty.version} - org.junit junit-bom From d406b276e8f7e074a915ce1ae323d8ef51d44a93 Mon Sep 17 00:00:00 2001 From: iliax Date: Tue, 4 Apr 2023 15:42:33 +0400 Subject: [PATCH 2/6] explicit spring security dependency removed --- pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pom.xml b/pom.xml index 5a3f759f36d..b314863a563 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,6 @@ 2.13.9 2.0 3.0.5 - 6.0.2 1.0.0 0.1.15 0.1.23 @@ -109,13 +108,6 @@ pom import - - org.springframework.security - spring-security-bom - ${spring-security.version} - pom - import - com.fasterxml.jackson jackson-bom From a60ec098e99f76e57e7298e1df730d30ecbc6412 Mon Sep 17 00:00:00 2001 From: iliax Date: Wed, 5 Apr 2023 12:40:08 +0400 Subject: [PATCH 3/6] openapi plugin updated to 6.5 --- .../ui/client/RetryingKafkaConnectClient.java | 212 +++++++++++++++--- .../ui/controller/KafkaConnectController.java | 10 +- .../kafka/ui/service/KafkaConnectService.java | 8 +- .../service/analyze/TopicAnalysisStats.java | 6 +- kafka-ui-contract/pom.xml | 36 +-- .../main/resources/swagger/kafka-ui-api.yaml | 4 + pom.xml | 4 +- 7 files changed, 205 insertions(+), 75 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/RetryingKafkaConnectClient.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/RetryingKafkaConnectClient.java index 5ec5a779d3c..74b9485008e 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/RetryingKafkaConnectClient.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/client/RetryingKafkaConnectClient.java @@ -6,7 +6,13 @@ import com.provectus.kafka.ui.connect.ApiClient; import com.provectus.kafka.ui.connect.api.KafkaConnectClientApi; import com.provectus.kafka.ui.connect.model.Connector; +import com.provectus.kafka.ui.connect.model.ConnectorPlugin; +import com.provectus.kafka.ui.connect.model.ConnectorPluginConfigValidationResponse; +import com.provectus.kafka.ui.connect.model.ConnectorStatus; +import com.provectus.kafka.ui.connect.model.ConnectorTask; +import com.provectus.kafka.ui.connect.model.ConnectorTopics; import com.provectus.kafka.ui.connect.model.NewConnector; +import com.provectus.kafka.ui.connect.model.TaskStatus; import com.provectus.kafka.ui.exception.KafkaConnectConflictReponseException; import com.provectus.kafka.ui.exception.ValidationException; import com.provectus.kafka.ui.util.WebClientConfigurator; @@ -15,11 +21,7 @@ import java.util.Map; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.util.MultiValueMap; +import org.springframework.http.ResponseEntity; import org.springframework.util.unit.DataSize; import org.springframework.web.client.RestClientException; import org.springframework.web.reactive.function.client.WebClient; @@ -79,6 +81,176 @@ public Mono setConnectorConfig(String connectorName, Map> createConnectorWithHttpInfo(NewConnector newConnector) + throws WebClientResponseException { + return withRetryOnConflict(super.createConnectorWithHttpInfo(newConnector)); + } + + @Override + public Mono deleteConnector(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.deleteConnector(connectorName)); + } + + @Override + public Mono> deleteConnectorWithHttpInfo(String connectorName) + throws WebClientResponseException { + return withRetryOnConflict(super.deleteConnectorWithHttpInfo(connectorName)); + } + + + @Override + public Mono getConnector(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.getConnector(connectorName)); + } + + @Override + public Mono> getConnectorWithHttpInfo(String connectorName) + throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorWithHttpInfo(connectorName)); + } + + @Override + public Mono> getConnectorConfig(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorConfig(connectorName)); + } + + @Override + public Mono>> getConnectorConfigWithHttpInfo(String connectorName) + throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorConfigWithHttpInfo(connectorName)); + } + + @Override + public Flux getConnectorPlugins() throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorPlugins()); + } + + @Override + public Mono>> getConnectorPluginsWithHttpInfo() + throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorPluginsWithHttpInfo()); + } + + @Override + public Mono getConnectorStatus(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorStatus(connectorName)); + } + + @Override + public Mono> getConnectorStatusWithHttpInfo(String connectorName) + throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorStatusWithHttpInfo(connectorName)); + } + + @Override + public Mono getConnectorTaskStatus(String connectorName, Integer taskId) + throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorTaskStatus(connectorName, taskId)); + } + + @Override + public Mono> getConnectorTaskStatusWithHttpInfo(String connectorName, Integer taskId) + throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorTaskStatusWithHttpInfo(connectorName, taskId)); + } + + @Override + public Flux getConnectorTasks(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorTasks(connectorName)); + } + + @Override + public Mono>> getConnectorTasksWithHttpInfo(String connectorName) + throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorTasksWithHttpInfo(connectorName)); + } + + @Override + public Mono> getConnectorTopics(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorTopics(connectorName)); + } + + @Override + public Mono>> getConnectorTopicsWithHttpInfo(String connectorName) + throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorTopicsWithHttpInfo(connectorName)); + } + + @Override + public Flux getConnectors(String search) throws WebClientResponseException { + return withRetryOnConflict(super.getConnectors(search)); + } + + @Override + public Mono>> getConnectorsWithHttpInfo(String search) throws WebClientResponseException { + return withRetryOnConflict(super.getConnectorsWithHttpInfo(search)); + } + + @Override + public Mono pauseConnector(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.pauseConnector(connectorName)); + } + + @Override + public Mono> pauseConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { + return withRetryOnConflict(super.pauseConnectorWithHttpInfo(connectorName)); + } + + @Override + public Mono restartConnector(String connectorName, Boolean includeTasks, Boolean onlyFailed) + throws WebClientResponseException { + return withRetryOnConflict(super.restartConnector(connectorName, includeTasks, onlyFailed)); + } + + @Override + public Mono> restartConnectorWithHttpInfo(String connectorName, Boolean includeTasks, + Boolean onlyFailed) throws WebClientResponseException { + return withRetryOnConflict(super.restartConnectorWithHttpInfo(connectorName, includeTasks, onlyFailed)); + } + + @Override + public Mono restartConnectorTask(String connectorName, Integer taskId) throws WebClientResponseException { + return withRetryOnConflict(super.restartConnectorTask(connectorName, taskId)); + } + + @Override + public Mono> restartConnectorTaskWithHttpInfo(String connectorName, Integer taskId) + throws WebClientResponseException { + return withRetryOnConflict(super.restartConnectorTaskWithHttpInfo(connectorName, taskId)); + } + + @Override + public Mono resumeConnector(String connectorName) throws WebClientResponseException { + return super.resumeConnector(connectorName); + } + + @Override + public Mono> resumeConnectorWithHttpInfo(String connectorName) + throws WebClientResponseException { + return withRetryOnConflict(super.resumeConnectorWithHttpInfo(connectorName)); + } + + @Override + public Mono> setConnectorConfigWithHttpInfo(String connectorName, + Map requestBody) + throws WebClientResponseException { + return withRetryOnConflict(super.setConnectorConfigWithHttpInfo(connectorName, requestBody)); + } + + @Override + public Mono validateConnectorPluginConfig(String pluginName, + Map requestBody) + throws WebClientResponseException { + return withRetryOnConflict(super.validateConnectorPluginConfig(pluginName, requestBody)); + } + + @Override + public Mono> validateConnectorPluginConfigWithHttpInfo( + String pluginName, Map requestBody) throws WebClientResponseException { + return withRetryOnConflict(super.validateConnectorPluginConfigWithHttpInfo(pluginName, requestBody)); + } + private static class RetryingApiClient extends ApiClient { public RetryingApiClient(ConnectCluster config, @@ -108,35 +280,5 @@ public static WebClient buildWebClient(DataSize maxBuffSize, .configureBufferSize(maxBuffSize) .build(); } - - @Override - public Mono invokeAPI(String path, HttpMethod method, Map pathParams, - MultiValueMap queryParams, Object body, - HttpHeaders headerParams, - MultiValueMap cookieParams, - MultiValueMap formParams, List accept, - MediaType contentType, String[] authNames, - ParameterizedTypeReference returnType) - throws RestClientException { - return withRetryOnConflict( - super.invokeAPI(path, method, pathParams, queryParams, body, headerParams, cookieParams, - formParams, accept, contentType, authNames, returnType) - ); - } - - @Override - public Flux invokeFluxAPI(String path, HttpMethod method, Map pathParams, - MultiValueMap queryParams, Object body, - HttpHeaders headerParams, - MultiValueMap cookieParams, - MultiValueMap formParams, - List accept, MediaType contentType, - String[] authNames, ParameterizedTypeReference returnType) - throws RestClientException { - return withRetryOnConflict( - super.invokeFluxAPI(path, method, pathParams, queryParams, body, headerParams, - cookieParams, formParams, accept, contentType, authNames, returnType) - ); - } } } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KafkaConnectController.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KafkaConnectController.java index 9ffd901c078..080c6020f9b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KafkaConnectController.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/controller/KafkaConnectController.java @@ -149,10 +149,9 @@ public Mono>> getConnectorConfig(String clust } @Override - public Mono> setConnectorConfig(String clusterName, - String connectName, + public Mono> setConnectorConfig(String clusterName, String connectName, String connectorName, - @Valid Mono requestBody, + Mono> requestBody, ServerWebExchange exchange) { Mono validateAccess = accessControlService.validateAccess(AccessContext.builder() @@ -164,8 +163,7 @@ public Mono> setConnectorConfig(String clusterName, return validateAccess.then( kafkaConnectService .setConnectorConfig(getCluster(clusterName), connectName, connectorName, requestBody) - .map(ResponseEntity::ok) - ); + .map(ResponseEntity::ok)); } @Override @@ -242,7 +240,7 @@ public Mono>> getConnectorPlugins( @Override public Mono> validateConnectorPluginConfig( - String clusterName, String connectName, String pluginName, @Valid Mono requestBody, + String clusterName, String connectName, String pluginName, @Valid Mono> requestBody, ServerWebExchange exchange) { return kafkaConnectService .validateConnectorPluginConfig( diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java index 163732fae9f..d07ef7ed2dd 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaConnectService.java @@ -225,11 +225,11 @@ public Mono> getConnectorConfig(KafkaCluster cluster, String } public Mono setConnectorConfig(KafkaCluster cluster, String connectName, - String connectorName, Mono requestBody) { + String connectorName, Mono> requestBody) { return api(cluster, connectName) .mono(c -> requestBody - .flatMap(body -> c.setConnectorConfig(connectorName, (Map) body)) + .flatMap(body -> c.setConnectorConfig(connectorName, body)) .map(kafkaConnectMapper::fromClient)); } @@ -298,12 +298,12 @@ public Flux getConnectorPlugins(KafkaCluster cluster, } public Mono validateConnectorPluginConfig( - KafkaCluster cluster, String connectName, String pluginName, Mono requestBody) { + KafkaCluster cluster, String connectName, String pluginName, Mono> requestBody) { return api(cluster, connectName) .mono(client -> requestBody .flatMap(body -> - client.validateConnectorPluginConfig(pluginName, (Map) body)) + client.validateConnectorPluginConfig(pluginName, body)) .map(kafkaConnectMapper::fromClient) ); } diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java index 2d8e0dc38f8..d5b4400807d 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/analyze/TopicAnalysisStats.java @@ -2,7 +2,7 @@ import com.provectus.kafka.ui.model.TopicAnalysisSizeStatsDTO; import com.provectus.kafka.ui.model.TopicAnalysisStatsDTO; -import com.provectus.kafka.ui.model.TopicAnalysisStatsHourlyMsgCountsDTO; +import com.provectus.kafka.ui.model.TopicAnalysisStatsHourlyMsgCountsInnerDTO; import java.time.Duration; import java.time.Instant; import java.util.Comparator; @@ -78,10 +78,10 @@ void apply(ConsumerRecord rec) { } } - List toDto() { + List toDto() { return hourlyStats.entrySet().stream() .sorted(Comparator.comparingLong(Map.Entry::getKey)) - .map(e -> new TopicAnalysisStatsHourlyMsgCountsDTO() + .map(e -> new TopicAnalysisStatsHourlyMsgCountsInnerDTO() .hourStart(e.getKey()) .count(e.getValue())) .collect(Collectors.toList()); diff --git a/kafka-ui-contract/pom.xml b/kafka-ui-contract/pom.xml index b585833875e..f99f20d3d81 100644 --- a/kafka-ui-contract/pom.xml +++ b/kafka-ui-contract/pom.xml @@ -27,31 +27,25 @@ spring-boot-starter-validation - io.swagger - swagger-annotations - ${swagger-annotations.version} + io.swagger.core.v3 + swagger-integration-jakarta + 2.2.8 org.openapitools jackson-databind-nullable - ${jackson-databind-nullable.version} + 0.2.4 - com.google.code.findbugs - jsr305 - 3.0.2 - provided + jakarta.annotation + jakarta.annotation-api + 2.1.1 javax.annotation javax.annotation-api 1.3.2 - - javax.validation - validation-api - 2.0.1.Final - @@ -81,6 +75,7 @@ webclient true java8 + true @@ -90,8 +85,7 @@ generate - ${project.basedir}/src/main/resources/swagger/kafka-ui-api.yaml - + ${project.basedir}/src/main/resources/swagger/kafka-ui-api.yaml ${project.build.directory}/generated-sources/api spring DTO @@ -99,14 +93,12 @@ com.provectus.kafka.ui.model com.provectus.kafka.ui.api kafka-ui-contract - true - true true true true - + true java8 @@ -126,15 +118,13 @@ java false false - com.provectus.kafka.ui.connect.model com.provectus.kafka.ui.connect.api kafka-connect-client - true webclient - + true true java8 @@ -152,15 +142,13 @@ java false false - com.provectus.kafka.ui.sr.model com.provectus.kafka.ui.sr.api kafka-sr-client - true webclient - + true true java8 diff --git a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml index fe9723b2dd3..d3b9f331366 100644 --- a/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml +++ b/kafka-ui-contract/src/main/resources/swagger/kafka-ui-api.yaml @@ -2387,6 +2387,10 @@ components: - UNKNOWN ConsumerGroup: + discriminator: + propertyName: inherit + mapping: + details: "#/components/schemas/ConsumerGroupDetails" type: object properties: groupId: diff --git a/pom.xml b/pom.xml index b314863a563..a82dc723916 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,6 @@ 3.1.0 3.0.13 2.14.0 - 0.2.4 3.3.1 1.4.2.Final 1.18.24 @@ -59,9 +58,8 @@ 3.10.1 3.2.0 2.22.2 - 4.3.0 + 6.5.0 1.2.32 - 1.6.0 From bbbc6e7c06364198d13ef9b3dd76b51af60e8c97 Mon Sep 17 00:00:00 2001 From: iliax Date: Wed, 5 Apr 2023 13:03:04 +0400 Subject: [PATCH 4/6] Some javax.annotation imports migrated to jakarta.annotation --- .../java/com/provectus/kafka/ui/config/ClustersProperties.java | 2 +- .../com/provectus/kafka/ui/config/auth/OAuthProperties.java | 2 +- .../provectus/kafka/ui/service/rbac/AccessControlService.java | 2 +- .../com/provectus/kafka/ui/service/ksql/KsqlServiceV2Test.java | 3 --- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java index 919e0633e4f..15436c1cd8b 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/ClustersProperties.java @@ -1,6 +1,7 @@ package com.provectus.kafka.ui.config; import com.provectus.kafka.ui.model.MetricsConfig; +import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -8,7 +9,6 @@ import java.util.Map; import java.util.Set; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthProperties.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthProperties.java index f79d217fa79..a76403bf707 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthProperties.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/config/auth/OAuthProperties.java @@ -1,9 +1,9 @@ package com.provectus.kafka.ui.config.auth; +import jakarta.annotation.PostConstruct; import java.util.HashMap; import java.util.Map; import java.util.Set; -import javax.annotation.PostConstruct; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.util.Assert; diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java index ee17d211117..3178feae34c 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/AccessControlService.java @@ -21,6 +21,7 @@ import com.provectus.kafka.ui.service.rbac.extractor.GoogleAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.LdapAuthorityExtractor; import com.provectus.kafka.ui.service.rbac.extractor.ProviderAuthorityExtractor; +import jakarta.annotation.PostConstruct; import java.util.Collections; import java.util.List; import java.util.Set; @@ -28,7 +29,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; diff --git a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlServiceV2Test.java b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlServiceV2Test.java index b4a48d38790..afa3700c0fa 100644 --- a/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlServiceV2Test.java +++ b/kafka-ui-api/src/test/java/com/provectus/kafka/ui/service/ksql/KsqlServiceV2Test.java @@ -15,7 +15,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.springframework.util.unit.DataSize; import org.testcontainers.utility.DockerImageName; class KsqlServiceV2Test extends AbstractIntegrationTest { @@ -27,8 +26,6 @@ class KsqlServiceV2Test extends AbstractIntegrationTest { private static final Set STREAMS_TO_DELETE = new CopyOnWriteArraySet<>(); private static final Set TABLES_TO_DELETE = new CopyOnWriteArraySet<>(); - private static final DataSize maxBuffSize = DataSize.ofMegabytes(20); - @BeforeAll static void init() { KSQL_DB.start(); From 34a2041b2039b9021ed1b3cd1d185c80370f6be3 Mon Sep 17 00:00:00 2001 From: iliax Date: Wed, 5 Apr 2023 14:58:37 +0400 Subject: [PATCH 5/6] base container sha specified --- kafka-ui-api/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kafka-ui-api/Dockerfile b/kafka-ui-api/Dockerfile index 270908fdfe1..d969ec76315 100644 --- a/kafka-ui-api/Dockerfile +++ b/kafka-ui-api/Dockerfile @@ -1,4 +1,5 @@ -FROM azul/zulu-openjdk-alpine:17-jre-headless +#FROM azul/zulu-openjdk-alpine:17-jre-headless +FROM azul/zulu-openjdk-alpine@sha256:a36679ac0d28cb835e2a8c00e1e0d95509c6c51c5081c7782b85edb1f37a771a RUN apk add --no-cache gcompat # need to make snappy codec work RUN addgroup -S kafkaui && adduser -S kafkaui -G kafkaui From 348350cf5b54acb7736b9843f0973b11d25cfa4a Mon Sep 17 00:00:00 2001 From: Roman Zabaluev Date: Fri, 7 Apr 2023 19:37:46 +0800 Subject: [PATCH 6/6] Update CognitoAuthorityExtractor --- .../ui/service/rbac/extractor/CognitoAuthorityExtractor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java index b148abdbd82..f7da0a19dba 100644 --- a/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java +++ b/kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/rbac/extractor/CognitoAuthorityExtractor.java @@ -3,12 +3,12 @@ import com.provectus.kafka.ui.model.rbac.Role; import com.provectus.kafka.ui.model.rbac.provider.Provider; import com.provectus.kafka.ui.service.rbac.AccessControlService; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; -import org.json.JSONArray; import org.springframework.security.oauth2.core.user.DefaultOAuth2User; import reactor.core.publisher.Mono; @@ -44,8 +44,7 @@ public Mono> extract(AccessControlService acs, Object value, Map groups = principal.getAttribute(COGNITO_GROUPS_ATTRIBUTE_NAME); if (groups == null) { log.debug("Cognito groups param is not present"); return Mono.just(groupsByUsername);