connectors-ci: fix postgres integration testing (#25942)

.github/workflows/connector_integration_test_single_dagger.yml

@@ -64,6 +64,8 @@ jobs:
           CI_GIT_REVISION: ${{ github.sha }}
           CI_CONTEXT: "manual"
           CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
+          S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
       - name: Run airbyte-ci connectors test [PULL REQUESTS]
         if: github.event_name == 'pull_request'
         run: |
@@ -88,3 +90,5 @@ jobs:
           CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }}
           CI_CONTEXT: "pull_request"
           CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
+          S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}

.github/workflows/connector_nightly_builds_dagger.yml

@@ -68,3 +68,5 @@ jobs:
           CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }}
           CI_CONTEXT: "nightly_builds"
           CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
+          S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java

@@ -12,14 +12,11 @@
 public class HostPortResolver {
 
   public static String resolveHost(GenericContainer container) {
-    return System.getProperty("os.name").toLowerCase().startsWith("mac")
-        ? getIpAddress(container)
-        : container.getHost();
+    return getIpAddress(container);
   }
 
   public static int resolvePort(GenericContainer container) {
-    return System.getProperty("os.name").toLowerCase().startsWith("mac") ? (Integer) container.getExposedPorts().get(0)
-        : container.getFirstMappedPort();
+    return (Integer) container.getExposedPorts().stream().findFirst().get();
   }
 
   public static String resolveIpAddress(GenericContainer container) {

airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java

@@ -17,6 +17,7 @@
 import io.airbyte.db.factory.DatabaseDriver;
 import io.airbyte.integrations.base.ssh.SshHelpers;
 import io.airbyte.integrations.standardtest.source.TestDestinationEnv;
+import io.airbyte.integrations.util.HostPortResolver;
 import io.airbyte.protocol.models.Field;
 import io.airbyte.protocol.models.JsonSchemaType;
 import io.airbyte.protocol.models.v0.CatalogHelpers;
@@ -60,11 +61,10 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc
     final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder()
         .put("method", "Standard")
         .build());
-    final var containerOuterAddress = SshHelpers.getOuterContainerAddress(container);
-    final var containerInnerAddress = SshHelpers.getInnerContainerAddress(container);
+
     config = Jsons.jsonNode(ImmutableMap.builder()
-        .put("host", containerInnerAddress.left)
-        .put("port", containerInnerAddress.right)
+        .put("host", HostPortResolver.resolveHost(container))
+        .put("port", HostPortResolver.resolvePort(container))
         .put("database", container.getDatabaseName())
         .put("schemas", Jsons.jsonNode(List.of("public")))
         .put("username", "postgres")
@@ -79,8 +79,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc
         config.get("password").asText(),
         DatabaseDriver.POSTGRESQL.getDriverClassName(),
         String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(),
-            containerOuterAddress.left,
-            containerOuterAddress.right,
+            container.getHost(),
+            container.getFirstMappedPort(),
             config.get("database").asText()),
         SQLDialect.POSTGRES)) {
       final Database database = new Database(dslContext);

airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java

@@ -23,7 +23,7 @@ public class CdcInitialSnapshotPostgresSourceDatatypeTest extends AbstractPostgr
   private static final String SCHEMA_NAME = "test";
   private static final String SLOT_NAME_BASE = "debezium_slot";
   private static final String PUBLICATION = "publication";
-  private static final int INITIAL_WAITING_SECONDS = 5;
+  private static final int INITIAL_WAITING_SECONDS = 30;
 
   @Override
   protected Database setupDatabase() throws Exception {

airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java

@@ -45,7 +45,7 @@ public class CdcPostgresSourceAcceptanceTest extends AbstractPostgresSourceAccep
   private static final String STREAM_NAME = "id_and_name";
   private static final String STREAM_NAME2 = "starships";
   protected static final String PUBLICATION = "publication";
-  protected static final int INITIAL_WAITING_SECONDS = 5;
+  protected static final int INITIAL_WAITING_SECONDS = 30;
 
   protected PostgreSQLContainer<?> container;
   protected JsonNode config;
@@ -63,6 +63,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc
         .put("publication", PUBLICATION)
         .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS)
         .build());
+
+
     config = Jsons.jsonNode(ImmutableMap.builder()
         .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container))
         .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container))

airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java

@@ -30,7 +30,7 @@ public class CdcWalLogsPostgresSourceDatatypeTest extends AbstractPostgresSource
   private static final String SCHEMA_NAME = "test";
   private static final String SLOT_NAME_BASE = "debezium_slot";
   private static final String PUBLICATION = "publication";
-  private static final int INITIAL_WAITING_SECONDS = 15;
+  private static final int INITIAL_WAITING_SECONDS = 30;
   private JsonNode stateAfterFirstSync;
 
   @Override

airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java

@@ -63,6 +63,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc
 
     container = new PostgreSQLContainer<>("postgres:13-alpine");
     container.start();
+
     final String username = container.getUsername();
     final String password = container.getPassword();
     final List<String> schemas = List.of("public");
@@ -94,6 +95,7 @@ private JsonNode getConfig(final String username, final String password, final L
     final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder()
         .put("method", "Standard")
         .build());
+
     return Jsons.jsonNode(ImmutableMap.builder()
         .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container))
         .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container))

airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java

@@ -75,7 +75,7 @@ public class CdcPostgresSourceTest extends CdcSourceTest {
 
   protected static final String SLOT_NAME_BASE = "debezium_slot";
   protected static final String PUBLICATION = "publication";
-  protected static final int INITIAL_WAITING_SECONDS = 5;
+  protected static final int INITIAL_WAITING_SECONDS = 30;
   private PostgreSQLContainer<?> container;
 
   protected String dbName;

airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java

@@ -41,7 +41,7 @@ class PostgresCdcGetPublicizedTablesTest {
   private static final String SCHEMA_NAME = "public";
   private static final String PUBLICATION = "publication_test_12";
   private static final String REPLICATION_SLOT = "replication_slot_test_12";
-  protected static final int INITIAL_WAITING_SECONDS = 5;
+  protected static final int INITIAL_WAITING_SECONDS = 30;
   private static PostgreSQLContainer<?> container;
   private JsonNode config;
 

airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java

@@ -140,7 +140,7 @@ void testLsnCommitBehaviourProperty(final String commitBehaviour) {
         .put("replication_slot", "replication_slot")
         .put("publication", "PUBLICATION")
         .put("plugin", "pgoutput")
-        .put("initial_waiting_seconds", 5)
+        .put("initial_waiting_seconds", 30)
         .put("lsn_commit_behaviour", commitBehaviour)
         .build());
 

tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py

@@ -408,13 +408,24 @@ def with_gradle(
         .with_env_variable("VERSION", "23.0.1")
         .with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"])
         .with_exec(["mkdir", "/root/.gradle"])
-        .with_mounted_cache("/root/.gradle", root_gradle_cache, sharing=CacheSharingMode.LOCKED)
-        .with_exec(["rm", "-f", "/root/.gradle/caches/journal-1/journal-1.lock"])
         .with_exec(["mkdir", "/airbyte"])
         .with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include))
         .with_mounted_cache("/airbyte/.gradle", airbyte_gradle_cache, sharing=CacheSharingMode.LOCKED)
         .with_workdir("/airbyte")
     )
+    if context.is_ci:
+        openjdk_with_docker = (
+            openjdk_with_docker.with_env_variable("CI", "true")
+            .with_secret_variable(
+                "S3_BUILD_CACHE_ACCESS_KEY_ID", context.dagger_client.host().env_variable("S3_BUILD_CACHE_ACCESS_KEY_ID").secret()
+            )
+            .with_secret_variable(
+                "S3_BUILD_CACHE_SECRET_KEY", context.dagger_client.host().env_variable("S3_BUILD_CACHE_SECRET_KEY").secret()
+            )
+        )
+    else:
+        openjdk_with_docker = openjdk_with_docker.with_mounted_cache("/root/.gradle", root_gradle_cache, sharing=CacheSharingMode.LOCKED)
+
     if bind_to_docker_host:
         return with_bound_docker_host(context, openjdk_with_docker, shared_tmp_volume, docker_service_name=docker_service_name)
     else: