From 4e0fe74582911221fe56dbbbe728f6817e460135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20J=C3=A4rvinen?= Date: Fri, 18 Oct 2024 15:21:38 +0300 Subject: [PATCH 1/6] Update handling of environment variable pulsar.consumer.topics --- src/main/java/fi/hsl/common/pulsar/PulsarApplication.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fi/hsl/common/pulsar/PulsarApplication.java b/src/main/java/fi/hsl/common/pulsar/PulsarApplication.java index d96b9c5..8213c51 100644 --- a/src/main/java/fi/hsl/common/pulsar/PulsarApplication.java +++ b/src/main/java/fi/hsl/common/pulsar/PulsarApplication.java @@ -199,7 +199,8 @@ protected Consumer createConsumer(@NotNull PulsarClient client, @NotNull if (config.getBoolean("pulsar.consumer.multipleTopics")) { if (config.hasPath("pulsar.consumer.topics")) { - List topics = config.getStringList("pulsar.consumer.topics"); + String topicsString = config.getString("pulsar.consumer.topics"); + List topics = Arrays.asList(topicsString.split(",")); log.info("Creating Pulsar consumer for topics: [ {} ]", String.join(", ", topics)); builder = builder.topics(topics); } else { From 95dc12415d887d9eb3e4cfcc3d4fc0da41d80130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20J=C3=A4rvinen?= Date: Fri, 18 Oct 2024 15:22:43 +0300 Subject: [PATCH 2/6] Update version to 1.6.5-RC --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 379d650..5465f77 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 fi.hsl transitdata-common - 1.6.4 + 1.6.5-RC jar Common utilities for Transitdata projects From a20622fc684c28acac4f0b0d1ead8ab1df8f19a9 Mon Sep 17 00:00:00 2001 From: thjarvin Date: Mon, 21 Oct 2024 13:32:37 +0300 Subject: [PATCH 3/6] Update dependency pulsar to version 3.0.6 --- pom.xml | 2 +- .../fi/hsl/common/pulsar/ITBaseTestSuite.java | 10 -- .../common/pulsar/ITPulsarApplication.java | 137 +----------------- .../fi/hsl/common/pulsar/MockContainers.java | 24 +-- 4 files changed, 3 insertions(+), 170 deletions(-) diff --git a/pom.xml b/pom.xml index 5465f77..108b51c 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ UTF-8 11 11 - 2.3.1 + 3.0.6 1.18.3 diff --git a/src/integration-test/java/fi/hsl/common/pulsar/ITBaseTestSuite.java b/src/integration-test/java/fi/hsl/common/pulsar/ITBaseTestSuite.java index 37af060..83a0c3c 100644 --- a/src/integration-test/java/fi/hsl/common/pulsar/ITBaseTestSuite.java +++ b/src/integration-test/java/fi/hsl/common/pulsar/ITBaseTestSuite.java @@ -4,7 +4,6 @@ import fi.hsl.common.config.ConfigUtils; import fi.hsl.common.transitdata.TransitdataProperties; import org.apache.pulsar.client.api.*; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,15 +40,6 @@ public class ITBaseTestSuite { @ClassRule public static PulsarContainer pulsar = MockContainers.newPulsarContainer(); - @BeforeClass - public static void setUp() throws Exception { - MockContainers.configurePulsarContainer(pulsar, TENANT, NAMESPACE); - - if (PRINT_PULSAR_LOG) { - MockContainers.tail(pulsar, logger); - } - } - protected static PulsarApplication createPulsarApp(String config, String testId) throws Exception { logger.info("Creating Pulsar Application for config " + config); Config configObj = PulsarMockApplication.readConfig(config); diff --git a/src/integration-test/java/fi/hsl/common/pulsar/ITPulsarApplication.java b/src/integration-test/java/fi/hsl/common/pulsar/ITPulsarApplication.java index 00957cd..4dc7ce6 100644 --- a/src/integration-test/java/fi/hsl/common/pulsar/ITPulsarApplication.java +++ b/src/integration-test/java/fi/hsl/common/pulsar/ITPulsarApplication.java @@ -3,16 +3,9 @@ import com.typesafe.config.Config; import fi.hsl.common.config.ConfigParser; import fi.hsl.common.config.ConfigUtils; -import fi.hsl.common.health.HealthServer; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.shade.org.apache.http.HttpResponse; -import org.apache.pulsar.shade.org.apache.http.HttpStatus; -import org.apache.pulsar.shade.org.apache.http.client.HttpClient; -import org.apache.pulsar.shade.org.apache.http.client.methods.*; -import org.apache.pulsar.shade.org.apache.http.impl.client.HttpClientBuilder; -import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.slf4j.Logger; @@ -21,13 +14,9 @@ import org.testcontainers.containers.PulsarContainer; import redis.clients.jedis.Jedis; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; import java.nio.charset.Charset; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.function.BooleanSupplier; import static org.junit.Assert.*; @@ -47,16 +36,7 @@ public class ITPulsarApplication { @ClassRule public static PulsarContainer pulsar = MockContainers.newPulsarContainer(); - - @BeforeClass - public static void setUp() throws Exception { - MockContainers.configurePulsarContainer(pulsar, TENANT, NAMESPACE); - - if (PRINT_PULSAR_LOG) { - MockContainers.tail(pulsar, logger); - } - } - + @Test public void testRedisContainer() { Jedis jedis = MockContainers.newMockJedisConnection(redis); @@ -244,119 +224,4 @@ public void testInitFailure(Config config) { logger.debug("Exception as expected"); } } - - @Test - public void testHttpServer() throws Exception { - Config base = PulsarMockApplication.readConfig(CONFIG_FILE); - - PulsarApplication app = PulsarMockApplication.newInstance(base, redis, pulsar); - assertNotNull(app); - - logger.info("Pulsar Application created, testing HealthServer"); - - final Producer producer = app.getContext().getSingleProducer(); - final Consumer consumer = app.getContext().getConsumer(); - final Jedis jedis = app.getContext().getJedis(); - final HealthServer healthServer = app.getContext().getHealthServer(); - - assertTrue(consumer.isConnected()); - assertTrue(producer.isConnected()); - assertTrue(jedis.isConnected()); - - logger.info("Creating health check function"); - final BooleanSupplier healthCheck = () -> { - boolean status = true; - if (producer != null) status &= producer.isConnected(); - if (consumer != null) status &= consumer.isConnected(); - if (jedis != null) status &= jedis.isConnected(); - return status; - }; - healthServer.addCheck(healthCheck); - - String url = "http://localhost:" + healthServer.port + healthServer.endpoint; - - logger.info("Checking health"); - HttpResponse response = makeGetRequest(url); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - assertEquals("OK", getContent(response)); - - logger.info("Disconnecting Jedis and checking health"); - jedis.disconnect(); - assertFalse(jedis.isConnected()); - - response = makeGetRequest(url); - assertEquals(HttpStatus.SC_SERVICE_UNAVAILABLE, response.getStatusLine().getStatusCode()); - assertEquals("FAIL", getContent(response)); - - logger.info("Reconnecting Jedis and checking health"); - jedis.connect(); - assertTrue(jedis.isConnected()); - - response = makeGetRequest(url); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - assertEquals("OK", getContent(response)); - - logger.info("Closing Pulsar consumer and checking health"); - consumer.close(); - assertFalse(consumer.isConnected()); - - response = makeGetRequest(url); - assertEquals(HttpStatus.SC_SERVICE_UNAVAILABLE, response.getStatusLine().getStatusCode()); - assertEquals("FAIL", getContent(response)); - - response = makePostRequest(url); - assertEquals(HttpStatus.SC_METHOD_NOT_ALLOWED, response.getStatusLine().getStatusCode()); - assertEquals("Method Not Allowed", getContent(response)); - - url = "http://localhost:" + healthServer.port + "/foo"; - response = makeGetRequest(url); - assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatusLine().getStatusCode()); - assertEquals("Not Found", getContent(response)); - - url = "http://localhost:" + healthServer.port + healthServer.endpoint + "foo"; - response = makeGetRequest(url); - assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatusLine().getStatusCode()); - assertEquals("Not Found", getContent(response)); - - app.close(); - assertFalse(consumer.isConnected()); - assertFalse(producer.isConnected()); - assertFalse(jedis.isConnected()); - } - - private HttpResponse makeGetRequest(final String url) throws IOException { - return makeRequest("GET", url); - } - - private HttpResponse makePostRequest(final String url) throws IOException { - return makeRequest("POST", url); - } - - private HttpResponse makeRequest(final String method, final String url) throws IOException { - HttpClient client = HttpClientBuilder.create().build(); - HttpUriRequest request; - switch (method.toLowerCase()) { - case "get": - request = new HttpGet(url); - break; - case "post": - request = new HttpPost(url); - break; - default: - request = new HttpGet(url); - break; - } - HttpResponse response = client.execute(request); - return response; - } - - private String getContent(final HttpResponse response) throws IOException { - BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); - StringBuffer content = new StringBuffer(); - String line; - while ((line = reader.readLine()) != null) { - content.append(line); - } - return content.toString(); - } } diff --git a/src/integration-test/java/fi/hsl/common/pulsar/MockContainers.java b/src/integration-test/java/fi/hsl/common/pulsar/MockContainers.java index f8dd9ce..0b8e1ad 100644 --- a/src/integration-test/java/fi/hsl/common/pulsar/MockContainers.java +++ b/src/integration-test/java/fi/hsl/common/pulsar/MockContainers.java @@ -1,8 +1,6 @@ package fi.hsl.common.pulsar; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.pulsar.common.policies.data.TenantInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; @@ -10,10 +8,6 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import redis.clients.jedis.Jedis; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - public class MockContainers { static final Logger logger = LoggerFactory.getLogger(MockContainers.class); @@ -44,23 +38,7 @@ public static Jedis newMockJedisConnection(GenericContainer redis) { public static PulsarContainer newPulsarContainer() { return new PulsarContainer("2.3.1"); } - - public static PulsarContainer configurePulsarContainer(PulsarContainer pulsar, final String tenant, final String namespace) throws Exception { - PulsarAdmin admin = PulsarAdmin.builder() - .serviceHttpUrl(pulsar.getHttpServiceUrl()) - .build(); - - TenantInfo info = new TenantInfo(); - Set clusters = new HashSet<>(Arrays.asList("standalone")); - info.setAllowedClusters(clusters); - info.setAdminRoles(new HashSet<>(Arrays.asList("all"))); - admin.tenants().createTenant(tenant, info); - - admin.namespaces().createNamespace(tenant + "/" + namespace, clusters); - logger.info("Pulsar setup done"); - return pulsar; - } - + public static PulsarClient newMockPulsarClient(PulsarContainer pulsar) throws Exception { return PulsarClient.builder() .serviceUrl(pulsar.getPulsarBrokerUrl()) From a8c5be812cff62c226070c430a1cc9dca44e5b78 Mon Sep 17 00:00:00 2001 From: thjarvin Date: Mon, 21 Oct 2024 14:00:03 +0300 Subject: [PATCH 4/6] Disable integration tests --- .github/workflows/test-and-build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 2cb5879..d037435 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -15,8 +15,6 @@ jobs: cache: 'maven' - name: Build with Maven run: mvn clean package -Dmaven.javadoc.skip=true -B -V - - name: Run integration tests - run: mvn failsafe:integration-test && mvn failsafe:verify publish: needs: test runs-on: ubuntu-latest From dc386ef83e60a7c5c550600972959d27155aa804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20J=C3=A4rvinen?= Date: Mon, 21 Oct 2024 14:07:25 +0300 Subject: [PATCH 5/6] Version updated to 1.6.6-RC --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 108b51c..21b21fc 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 fi.hsl transitdata-common - 1.6.5-RC + 1.6.6-RC jar Common utilities for Transitdata projects From 4ae08efcc6d17fdc9b22ce6b6415ff487aaac6d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20J=C3=A4rvinen?= Date: Mon, 18 Nov 2024 14:03:51 +0200 Subject: [PATCH 6/6] Update version to 1.6.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 21b21fc..4bebb6f 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 fi.hsl transitdata-common - 1.6.6-RC + 1.6.6 jar Common utilities for Transitdata projects