Merge branch 'master' into test-parallelism-and-logging

.github/workflows/gradle.yml

@@ -89,6 +89,12 @@ jobs:
       - name: Install Pyenv
         if: steps.changes.outputs.gradlecheck_any_changed == 'true'
         run: python3 -m pip install virtualenv --user
+      - name: Docker login
+        # Some tests use testcontainers which pull images from DockerHub.
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
       - name: Run Gradle Check
         if: steps.changes.outputs.gradlecheck_any_changed == 'true'
         uses: burrunan/gradle-cache-action@v1
@@ -98,7 +104,6 @@ jobs:
           read-only: ${{ github.ref != 'refs/heads/master' }}
           # TODO: be able to remove the skipSlowTests property
           # TODO: remove the format task ASAP
-          # TODO: roll the CDK tasks into check
           arguments: --scan --no-daemon --no-watch-fs format check -DskipSlowTests=true
 
   # In case of self-hosted EC2 errors, remove this block.

airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle

@@ -2,6 +2,12 @@ plugins {
     id "java-library"
 }
 
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-try"
+    }
+}
+
 dependencies {
     annotationProcessor platform(libs.micronaut.bom)
     annotationProcessor libs.bundles.micronaut.annotation.processor
@@ -19,6 +25,7 @@ dependencies {
     implementation 'org.apache.ant:ant:1.10.10'
     implementation 'org.apache.commons:commons-text:1.10.0'
     implementation libs.bundles.datadog
+    implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2'
 
     implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-api')
     implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons')

airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle

@@ -5,6 +5,12 @@ plugins {
     id "java-library"
 }
 
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-deprecation"
+    }
+}
+
 def specFile = "$projectDir/src/main/openapi/config.yaml"
 
 def generate = tasks.register('generate')

airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle

@@ -1,3 +1,9 @@
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-unchecked"
+    }
+}
+
 dependencies {
     annotationProcessor libs.bundles.micronaut.annotation.processor
     testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor

airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle

@@ -3,6 +3,12 @@ plugins {
     id 'de.undercouch.download' version "5.4.0"
 }
 
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-varargs,-try,-deprecation"
+    }
+}
+
 dependencies {
     // Dependencies for this module should be specified in the top-level build.gradle. See readme for more explanation.
 

airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java

@@ -22,6 +22,7 @@ public class MoreIterators {
    * @param <T> type
    * @return iterator with all elements
    */
+  @SafeVarargs
   public static <T> Iterator<T> of(final T... elements) {
     return Arrays.asList(elements).iterator();
   }

airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle

@@ -5,6 +5,12 @@ plugins {
     id "com.github.eirnym.js2p" version "1.0"
 }
 
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-unchecked"
+    }
+}
+
 dependencies {
     annotationProcessor libs.bundles.micronaut.annotation.processor
     api libs.bundles.micronaut.annotation

airbyte-cdk/java/airbyte-cdk/core/build.gradle

@@ -1,4 +1,10 @@
 
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-cast,-unchecked"
+    }
+}
+
 configurations.all {
     resolutionStrategy {
         // TODO: Diagnose conflicting dependencies and remove these force overrides:
@@ -63,8 +69,7 @@ dependencies {
     implementation libs.hikaricp
     implementation libs.bundles.debezium.bundle
 
-    implementation libs.bundles.datadog
-    // implementation 'com.datadoghq:dd-trace-api'
+    api libs.bundles.datadog
     implementation 'org.apache.sshd:sshd-mina:2.8.0'
 
     implementation libs.testcontainers

airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

@@ -1 +1 @@
-version=0.3.0
+version=0.4.0

airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBufferTest.java

@@ -69,7 +69,7 @@ public void testGzipAvroFileWriter() throws Exception {
         "codec", "zstandard",
         "compression_level", 20,
         "include_checksum", true))));
-    runTest(new FileBuffer(AvroSerializedBuffer.DEFAULT_SUFFIX), 970L, 985L, config, getExpectedString());
+    runTest(new FileBuffer(AvroSerializedBuffer.DEFAULT_SUFFIX), 965L, 985L, config, getExpectedString());
   }
 
   @Test

airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle

@@ -1,3 +1,10 @@
+
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-deprecation"
+    }
+}
+
 dependencies {
     // Depends on core CDK classes (OK 👍)
     implementation project(':airbyte-cdk:java:airbyte-cdk:core')

airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle

@@ -17,6 +17,12 @@ plugins {
     id "java-test-fixtures"  // https://docs.gradle.org/current/userguide/java_testing.html#sec:java_test_fixtures
 }
 
+java {
+    compileJava {
+        options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked,-removal"
+    }
+}
+
 project.configurations {
     // From `base-debezium`:
     testFixturesImplementation.extendsFrom implementation

airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/AbstractStateManager.java

@@ -22,7 +22,7 @@
  * @param <T> The type associated with the state object managed by this manager.
  * @param <S> The type associated with the state object stored in the state managed by this manager.
  */
-public abstract class AbstractStateManager<T, S> implements StateManager<T, S> {
+public abstract class AbstractStateManager<T, S> implements StateManager {
 
   /**
    * The {@link CursorManager} responsible for keeping track of the current cursor value for each

airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManager.java

@@ -22,7 +22,7 @@
  * @param <T> The type of the state maintained by the manager.
  * @param <S> The type of the stream(s) stored within the state maintained by the manager.
  */
-public interface StateManager<T, S> {
+public interface StateManager {
 
   Logger LOGGER = LoggerFactory.getLogger(StateManager.class);
 

airbyte-ci/connectors/pipelines/README.md

@@ -408,7 +408,8 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
 | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
-| 2.5.3   | [#32090](https://github.com/airbytehq/airbyte/pull/32090)  | Do not cache `docker login`.                            |
+| 2.5.5   | [#32114](https://github.com/airbytehq/airbyte/pull/32114)  | Create cache mount for `/var/lib/docker` to store images in `dind` context.                            |
+| 2.5.4   | [#32090](https://github.com/airbytehq/airbyte/pull/32090)  | Do not cache `docker login`.                            |
 | 2.5.3   | [#31974](https://github.com/airbytehq/airbyte/pull/31974)  | Fix latest CDK install and pip cache mount on connector install.                            |
 | 2.5.2   | [#31871](https://github.com/airbytehq/airbyte/pull/31871)  | Deactivate PR comments, add HTML report links to the PR status when its ready.                            |
 | 2.5.1   | [#31774](https://github.com/airbytehq/airbyte/pull/31774)  | Add a docker configuration check on `airbyte-ci` startup.                                                 |

airbyte-ci/connectors/pipelines/pipelines/consts.py

@@ -45,6 +45,7 @@
 DOCKER_HOST_NAME = "global-docker-host"
 DOCKER_HOST_PORT = 2375
 DOCKER_TMP_VOLUME_NAME = "shared-tmp"
+DOCKER_VAR_LIB_VOLUME_NAME = "docker-cache"
 REPO = git.Repo(search_parent_directories=True)
 REPO_PATH = REPO.working_tree_dir
 STATIC_REPORT_PREFIX = "airbyte-ci"

airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py

@@ -9,7 +9,7 @@
 from dagger import Client, Container, File, Secret
 from pipelines import consts
 from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext
-from pipelines.consts import DOCKER_HOST_NAME, DOCKER_HOST_PORT, DOCKER_TMP_VOLUME_NAME
+from pipelines.consts import DOCKER_HOST_NAME, DOCKER_HOST_PORT, DOCKER_TMP_VOLUME_NAME, DOCKER_VAR_LIB_VOLUME_NAME
 from pipelines.helpers.utils import sh_dash_c
 
 
@@ -26,7 +26,8 @@ def with_global_dockerd_service(
         Container: The container running dockerd as a service
     """
     dockerd_container = (
-        dagger_client.container().from_(consts.DOCKER_DIND_IMAGE)
+        dagger_client.container()
+        .from_(consts.DOCKER_DIND_IMAGE)
         # We set this env var because we need to use a non-default zombie reaper setting.
         # The reason for this is that by default it will want to set its parent process ID to 1 when reaping.
         # This won't be possible because of container-ception: dind is running inside the dagger engine.
@@ -49,6 +50,7 @@ def with_global_dockerd_service(
         # Expose the docker host port.
         .with_exposed_port(DOCKER_HOST_PORT)
         # Mount the docker cache volumes.
+        .with_mounted_cache("/var/lib/docker", dagger_client.cache_volume(DOCKER_VAR_LIB_VOLUME_NAME))
         .with_mounted_cache("/tmp", dagger_client.cache_volume(DOCKER_TMP_VOLUME_NAME))
     )
     if docker_hub_username_secret and docker_hub_password_secret:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "2.5.4"
+version = "2.5.5"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-integrations/connectors/destination-weaviate/metadata.yaml

@@ -13,7 +13,7 @@ data:
   connectorSubtype: vectorstore
   connectorType: destination
   definitionId: 7b7d7a0d-954c-45a0-bcfc-39a634b97736
-  dockerImageTag: 0.2.6
+  dockerImageTag: 0.2.7
   dockerRepository: airbyte/destination-weaviate
   documentationUrl: https://docs.airbyte.com/integrations/destinations/weaviate
   githubIssueLabel: destination-weaviate

airbyte-integrations/connectors/destination-weaviate/setup.py

@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.41", "weaviate-client==3.23.2"]
+MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.41", "weaviate-client==3.25.2"]
 
 TEST_REQUIREMENTS = ["pytest~=6.2", "docker", "pytest-docker"]
 

airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml

@@ -28,6 +28,8 @@ acceptance_tests:
             bypass_reason: "Hourly reports are disabled, because sync is too long"
           - name: campaign_performance_report_hourly
             bypass_reason: "Hourly reports are disabled, because sync is too long"
+          - name: campaign_impression_performance_report_hourly
+            bypass_reason: "Empty report; hourly data fetched is limited to 180 days"
           - name: keyword_performance_report_hourly
             bypass_reason: "Hourly reports are disabled, because sync is too long"
           - name: geographic_performance_report_hourly

airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json

@@ -226,6 +226,46 @@
       "cursor_field": ["TimePeriod"],
       "destination_sync_mode": "append"
     },
+    {
+      "stream": {
+        "name": "campaign_impression_performance_report_hourly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "campaign_impression_performance_report_daily",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "campaign_impression_performance_report_weekly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "campaign_impression_performance_report_monthly",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental", "full_refresh"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["TimePeriod"],
+      "destination_sync_mode": "append"
+    },
     {
       "stream": {
         "name": "keyword_performance_report_hourly",