Revert "remove airbyte-commons-docker (#21756)"

This reverts commit 9c4100b.

.github/workflows/gradle.yml

@@ -708,6 +708,7 @@ jobs:
             ${{ github.workspace }}/airbyte-bootloader/build/reports/jacoco/test/jacocoTestReport.xml,
             ${{ github.workspace }}/airbyte-commons/build/reports/jacoco/test/jacocoTestReport.xml,
             ${{ github.workspace }}/airbyte-commons-cli/build/reports/jacoco/test/jacocoTestReport.xml,
+            ${{ github.workspace }}/airbyte-commons-docker/build/reports/jacoco/test/jacocoTestReport.xml,
             ${{ github.workspace }}/airbyte-commons-protocol/build/reports/jacoco/test/jacocoTestReport.xml,
             ${{ github.workspace }}/airbyte-commons-temporal/build/reports/jacoco/test/jacocoTestReport.xml,
             ${{ github.workspace }}/airbyte-commons-worker/build/reports/jacoco/test/jacocoTestReport.xml,
@@ -1126,7 +1127,7 @@ jobs:
   helm-acceptance-test:
     name: "Platform: Acceptance Tests (Helm)"
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
-    needs: [ start-helm-acceptance-test-runner ] # required to start the main job when the runner is ready
+    needs: [start-helm-acceptance-test-runner] # required to start the main job when the runner is ready
     runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
     # this is the label of the runner
     environment: more-secrets

airbyte-commons-docker/build.gradle

@@ -0,0 +1,13 @@
+plugins {
+    id "java-library"
+}
+
+dependencies {
+    implementation 'org.apache.commons:commons-compress:1.20'
+    implementation 'com.github.docker-java:docker-java:3.2.8'
+    implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8'
+
+    testImplementation 'org.apache.commons:commons-lang3:3.11'
+}
+
+Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-commons-docker/readme.md

@@ -0,0 +1,3 @@
+# airbyte-commons-docker
+
+This module contains common helpers for interacting with Docker and Docker images from Java.

airbyte-commons-docker/src/main/java/io/airbyte/commons/docker/DockerUtils.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.commons.docker;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.BuildImageResultCallback;
+import com.github.dockerjava.core.DefaultDockerClientConfig;
+import com.github.dockerjava.core.DockerClientConfig;
+import com.github.dockerjava.core.DockerClientImpl;
+import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
+import com.github.dockerjava.transport.DockerHttpClient;
+import java.io.File;
+import java.util.Set;
+
+public class DockerUtils {
+
+  private static final DockerClientConfig CONFIG = DefaultDockerClientConfig.createDefaultConfigBuilder().build();
+  private static final DockerHttpClient HTTP_CLIENT = new ApacheDockerHttpClient.Builder()
+      .dockerHost(CONFIG.getDockerHost())
+      .sslConfig(CONFIG.getSSLConfig())
+      .maxConnections(100)
+      .build();
+  private static final DockerClient DOCKER_CLIENT = DockerClientImpl.getInstance(CONFIG, HTTP_CLIENT);
+
+  public static String getTaggedImageName(final String dockerRepository, final String tag) {
+    return String.join(":", dockerRepository, tag);
+  }
+
+  public static String buildImage(final String dockerFilePath, final String tag) {
+    return DOCKER_CLIENT.buildImageCmd()
+        .withDockerfile(new File(dockerFilePath))
+        .withTags(Set.of(tag))
+        .exec(new BuildImageResultCallback())
+        .awaitImageId();
+  }
+
+}

airbyte-commons-docker/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.commons.docker;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+class DockerUtilsTest {
+
+  @Test
+  void testGetTaggedImageName() {
+    final String repository = "airbyte/repo";
+    final String tag = "12.3";
+    assertEquals("airbyte/repo:12.3", DockerUtils.getTaggedImageName(repository, tag));
+  }
+
+}

airbyte-commons-server/build.gradle

@@ -29,6 +29,7 @@ dependencies {
 
     implementation project(':airbyte-analytics')
     implementation project(':airbyte-api')
+    implementation project(':airbyte-commons-docker')
     implementation project(':airbyte-commons-temporal')
     implementation project(':airbyte-commons-worker')
     implementation project(':airbyte-config:init')
@@ -65,4 +66,4 @@ dependencies {
     testImplementation 'org.mockito:mockito-inline:4.7.0'
 }
 
-Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)
+Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java

@@ -17,6 +17,7 @@
 import io.airbyte.api.model.generated.PrivateDestinationDefinitionReadList;
 import io.airbyte.api.model.generated.ReleaseStage;
 import io.airbyte.api.model.generated.WorkspaceIdRequestBody;
+import io.airbyte.commons.docker.DockerUtils;
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.commons.server.ServerConstants;
 import io.airbyte.commons.server.converters.ApiPojoConverters;
@@ -31,11 +32,7 @@
 import io.airbyte.commons.version.AirbyteProtocolVersion;
 import io.airbyte.commons.version.AirbyteProtocolVersionRange;
 import io.airbyte.commons.version.Version;
-import io.airbyte.config.ActorDefinitionResourceRequirements;
-import io.airbyte.config.ActorType;
-import io.airbyte.config.Configs;
-import io.airbyte.config.EnvConfigs;
-import io.airbyte.config.StandardDestinationDefinition;
+import io.airbyte.config.*;
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.protocol.models.ConnectorSpecification;
@@ -298,7 +295,7 @@ public void deleteDestinationDefinition(final DestinationDefinitionIdRequestBody
 
   private ConnectorSpecification getSpecForImage(final String dockerRepository, final String imageTag, final boolean isCustomConnector)
       throws IOException {
-    final String imageName = dockerRepository + ":" + imageTag;
+    final String imageName = DockerUtils.getTaggedImageName(dockerRepository, imageTag);
     final SynchronousResponse<ConnectorSpecification> getSpecResponse = schedulerSynchronousClient.createGetSpecJob(imageName, isCustomConnector);
     return SpecFetcher.getSpecFromJob(getSpecResponse);
   }

airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java

@@ -41,6 +41,7 @@
 import io.airbyte.api.model.generated.StreamTransform;
 import io.airbyte.api.model.generated.StreamTransform.TransformTypeEnum;
 import io.airbyte.api.model.generated.SynchronousJobRead;
+import io.airbyte.commons.docker.DockerUtils;
 import io.airbyte.commons.enums.Enums;
 import io.airbyte.commons.features.FeatureFlags;
 import io.airbyte.commons.json.Jsons;
@@ -152,7 +153,7 @@ public CheckConnectionRead checkSourceConnectionFromSourceId(final SourceIdReque
       throws ConfigNotFoundException, IOException, JsonValidationException {
     final SourceConnection source = configRepository.getSourceConnection(sourceIdRequestBody.getSourceId());
     final StandardSourceDefinition sourceDef = configRepository.getStandardSourceDefinition(source.getSourceDefinitionId());
-    final String imageName = sourceDef.getDockerRepository() + ":" + sourceDef.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag());
     final boolean isCustomConnector = sourceDef.getCustom();
     final Version protocolVersion = new Version(sourceDef.getProtocolVersion());
 
@@ -176,7 +177,7 @@ public CheckConnectionRead checkSourceConnectionFromSourceCreate(final SourceCor
 
     final Version protocolVersion = new Version(sourceDef.getProtocolVersion());
 
-    final String imageName = sourceDef.getDockerRepository() + ":" + sourceDef.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag());
     final boolean isCustomConnector = sourceDef.getCustom();
     return reportConnectionStatus(synchronousSchedulerClient.createSourceCheckConnectionJob(source, imageName, protocolVersion, isCustomConnector));
   }
@@ -201,7 +202,7 @@ public CheckConnectionRead checkDestinationConnectionFromDestinationId(final Des
       throws ConfigNotFoundException, IOException, JsonValidationException {
     final DestinationConnection destination = configRepository.getDestinationConnection(destinationIdRequestBody.getDestinationId());
     final StandardDestinationDefinition destinationDef = configRepository.getStandardDestinationDefinition(destination.getDestinationDefinitionId());
-    final String imageName = destinationDef.getDockerRepository() + ":" + destinationDef.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(destinationDef.getDockerRepository(), destinationDef.getDockerImageTag());
     final boolean isCustomConnector = destinationDef.getCustom();
     final Version protocolVersion = new Version(destinationDef.getProtocolVersion());
     return reportConnectionStatus(
@@ -223,7 +224,7 @@ public CheckConnectionRead checkDestinationConnectionFromDestinationCreate(final
         .withDestinationDefinitionId(destinationConfig.getDestinationDefinitionId())
         .withConfiguration(partialConfig)
         .withWorkspaceId(destinationConfig.getWorkspaceId());
-    final String imageName = destDef.getDockerRepository() + ":" + destDef.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(destDef.getDockerRepository(), destDef.getDockerImageTag());
     final Version protocolVersion = new Version(destDef.getProtocolVersion());
     return reportConnectionStatus(
         synchronousSchedulerClient.createDestinationCheckConnectionJob(destination, imageName, protocolVersion, isCustomConnector));
@@ -249,7 +250,7 @@ public SourceDiscoverSchemaRead discoverSchemaForSourceFromSourceId(final Source
       throws ConfigNotFoundException, IOException, JsonValidationException {
     final SourceConnection source = configRepository.getSourceConnection(discoverSchemaRequestBody.getSourceId());
     final StandardSourceDefinition sourceDef = configRepository.getStandardSourceDefinition(source.getSourceDefinitionId());
-    final String imageName = sourceDef.getDockerRepository() + ":" + sourceDef.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag());
     final boolean isCustomConnector = sourceDef.getCustom();
 
     final String configHash = HASH_FUNCTION.hashBytes(Jsons.serialize(source.getConfiguration()).getBytes(
@@ -297,7 +298,7 @@ public SourceDiscoverSchemaRead discoverSchemaForSourceFromSourceCreate(final So
         sourceCreate.getConnectionConfiguration(),
         sourceDef.getSpec());
 
-    final String imageName = sourceDef.getDockerRepository() + ":" + sourceDef.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(sourceDef.getDockerRepository(), sourceDef.getDockerImageTag());
     final boolean isCustomConnector = sourceDef.getCustom();
     // todo (cgardens) - narrow the struct passed to the client. we are not setting fields that are
     // technically declared as required.

airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java

@@ -18,6 +18,7 @@
 import io.airbyte.api.model.generated.SourceDefinitionUpdate;
 import io.airbyte.api.model.generated.SourceRead;
 import io.airbyte.api.model.generated.WorkspaceIdRequestBody;
+import io.airbyte.commons.docker.DockerUtils;
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.commons.server.ServerConstants;
 import io.airbyte.commons.server.converters.ApiPojoConverters;
@@ -32,11 +33,7 @@
 import io.airbyte.commons.version.AirbyteProtocolVersion;
 import io.airbyte.commons.version.AirbyteProtocolVersionRange;
 import io.airbyte.commons.version.Version;
-import io.airbyte.config.ActorDefinitionResourceRequirements;
-import io.airbyte.config.ActorType;
-import io.airbyte.config.Configs;
-import io.airbyte.config.EnvConfigs;
-import io.airbyte.config.StandardSourceDefinition;
+import io.airbyte.config.*;
 import io.airbyte.config.persistence.ConfigNotFoundException;
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.protocol.models.ConnectorSpecification;
@@ -299,7 +296,7 @@ public void deleteSourceDefinition(final SourceDefinitionIdRequestBody sourceDef
 
   private ConnectorSpecification getSpecForImage(final String dockerRepository, final String imageTag, final boolean isCustomConnector)
       throws IOException {
-    final String imageName = dockerRepository + ":" + imageTag;
+    final String imageName = DockerUtils.getTaggedImageName(dockerRepository, imageTag);
     final SynchronousResponse<ConnectorSpecification> getSpecResponse = schedulerSynchronousClient.createGetSpecJob(imageName, isCustomConnector);
     return SpecFetcher.getSpecFromJob(getSpecResponse);
   }

airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java

@@ -29,6 +29,7 @@
 import io.airbyte.api.model.generated.PrivateDestinationDefinitionReadList;
 import io.airbyte.api.model.generated.ReleaseStage;
 import io.airbyte.api.model.generated.WorkspaceIdRequestBody;
+import io.airbyte.commons.docker.DockerUtils;
 import io.airbyte.commons.json.Jsons;
 import io.airbyte.commons.server.errors.IdNotFoundKnownException;
 import io.airbyte.commons.server.errors.UnsupportedProtocolVersionException;
@@ -381,7 +382,7 @@ void testCreateDestinationDefinitionShouldCheckProtocolVersion() throws URISynta
     final String invalidProtocolVersion = "121.5.6";
     final StandardDestinationDefinition destination = generateDestinationDefinition();
     destination.getSpec().setProtocolVersion(invalidProtocolVersion);
-    final String imageName = destination.getDockerRepository() + ":" + destination.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag());
 
     when(uuidSupplier.get()).thenReturn(destination.getDestinationDefinitionId());
     when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>(
@@ -416,7 +417,7 @@ void testCreateDestinationDefinitionShouldCheckProtocolVersion() throws URISynta
   @DisplayName("createCustomDestinationDefinition should correctly create a destinationDefinition")
   void testCreateCustomDestinationDefinition() throws URISyntaxException, IOException, JsonValidationException {
     final StandardDestinationDefinition destination = generateDestinationDefinition();
-    final String imageName = destination.getDockerRepository() + ":" + destination.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag());
 
     when(uuidSupplier.get()).thenReturn(destination.getDestinationDefinitionId());
     when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>(
@@ -473,7 +474,7 @@ void testCreateCustomDestinationDefinitionWithInvalidProtocol() throws URISyntax
     final String invalidProtocol = "122.1.22";
     final StandardDestinationDefinition destination = generateDestinationDefinition();
     destination.getSpec().setProtocolVersion(invalidProtocol);
-    final String imageName = destination.getDockerRepository() + ":" + destination.getDockerImageTag();
+    final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag());
 
     when(uuidSupplier.get()).thenReturn(destination.getDestinationDefinitionId());
     when(schedulerSynchronousClient.createGetSpecJob(imageName, true)).thenReturn(new SynchronousResponse<>(
@@ -520,7 +521,7 @@ void testUpdateDestination() throws ConfigNotFoundException, IOException, JsonVa
     assertNotEquals(newDockerImageTag, currentTag);
     assertNotEquals(newProtocolVersion, currentDestination.getProtocolVersion());
 
-    final String newImageName = destinationDefinition.getDockerRepository() + ":" + newDockerImageTag;
+    final String newImageName = DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), newDockerImageTag);
     final ConnectorSpecification newSpec = new ConnectorSpecification()
         .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2")))
         .withProtocolVersion(newProtocolVersion);
@@ -556,7 +557,7 @@ void testOutOfProtocolRangeUpdateDestination() throws ConfigNotFoundException, I
     assertNotEquals(newDockerImageTag, currentTag);
     assertNotEquals(newProtocolVersion, currentDestination.getProtocolVersion());
 
-    final String newImageName = destinationDefinition.getDockerRepository() + ":" + newDockerImageTag;
+    final String newImageName = DockerUtils.getTaggedImageName(destinationDefinition.getDockerRepository(), newDockerImageTag);
     final ConnectorSpecification newSpec = new ConnectorSpecification()
         .withConnectionSpecification(Jsons.jsonNode(ImmutableMap.of("foo2", "bar2")))
         .withProtocolVersion(newProtocolVersion);

airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java

@@ -47,6 +47,7 @@
 import io.airbyte.api.model.generated.SourceUpdate;
 import io.airbyte.api.model.generated.StreamTransform;
 import io.airbyte.api.model.generated.StreamTransform.TransformTypeEnum;
+import io.airbyte.commons.docker.DockerUtils;
 import io.airbyte.commons.enums.Enums;
 import io.airbyte.commons.features.EnvVariableFeatureFlags;
 import io.airbyte.commons.json.Jsons;
@@ -102,12 +103,12 @@ class SchedulerHandlerTest {
 
   private static final String SOURCE_DOCKER_REPO = "srcimage";
   private static final String SOURCE_DOCKER_TAG = "tag";
-  private static final String SOURCE_DOCKER_IMAGE = SOURCE_DOCKER_REPO + ":" + SOURCE_DOCKER_TAG;
+  private static final String SOURCE_DOCKER_IMAGE = DockerUtils.getTaggedImageName(SOURCE_DOCKER_REPO, SOURCE_DOCKER_TAG);
   private static final String SOURCE_PROTOCOL_VERSION = "0.4.5";
 
   private static final String DESTINATION_DOCKER_REPO = "dstimage";
   private static final String DESTINATION_DOCKER_TAG = "tag";
-  private static final String DESTINATION_DOCKER_IMAGE = DESTINATION_DOCKER_REPO + ":" + DESTINATION_DOCKER_TAG;
+  private static final String DESTINATION_DOCKER_IMAGE = DockerUtils.getTaggedImageName(DESTINATION_DOCKER_REPO, DESTINATION_DOCKER_TAG);
   private static final String DESTINATION_PROTOCOL_VERSION = "0.7.9";
   private static final String NAME = "name";
   private static final String DOGS = "dogs";