source-mssql: convert to bulk CDK

.github/ISSUE_TEMPLATE/8-documentation.yaml

@@ -1,16 +1,17 @@
-name: 📚 Request or report a problem with documenation
-description: For all docs typos, tutorials requests, etc.
-labels: [type/enhancement, area/documentation needs-triage]
+name: 📚 Request or report a problem with documentation
+description: Typos, tutorial requests, and everything in between.
+title: "[Docs]: "
+labels: [area/documentation, needs-triage]
 body:
   - type: input
     id: url
     attributes:
       label: Page URL
-      description: Paste the URL with problem when reporting an issue
+      description: If this is an issue with a specific page, paste the URL here.
       placeholder: https://docs.airbyte.com/connector-development/connector-specification-reference
   - type: textarea
     id: description
     attributes:
       label: Description
       description: >-
-        It can be as pedantic as a typo or as titanic as an entire tutorial request.
+        Please provide as much detail as possible to ensure we can resolve the issue properly.

.github/workflows/bump-version-command.yml

@@ -17,10 +17,10 @@ on:
         default: "patch"
 
       changelog:
-        description: "The comment to add to the changelog."
+        description: "Optional. The comment to add to the changelog. If not provided, the PR title will be used."
         required: false
-        # TODO: We could infer the changelog string from the PR description!
-        default: "Bumped automatically in the pull request, please see PR description"
+        # If not provided, we'll use the PR title as the changelog
+        default: ""
 
       # These must be declared, but they are unused and ignored.
       # TODO: Infer 'repo' and 'gitref' from PR number on other workflows, so we can remove these.
@@ -52,6 +52,7 @@ jobs:
           PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }})
           echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT
           echo "branch=$(echo "$PR_JSON" | jq -r .head.ref)" >> $GITHUB_OUTPUT
+          echo "pr_title=$(echo "$PR_JSON" | jq -r .title)" >> $GITHUB_OUTPUT
           echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
 
       - name: Checkout Airbyte
@@ -78,6 +79,14 @@ jobs:
 
             [1]: ${{ steps.job-vars.outputs.run-url }}
 
+      - name: Log changelog source
+        run: |
+          if [ -n "${{ github.event.inputs.changelog }}" ]; then
+            echo "Using user-provided changelog: ${{ github.event.inputs.changelog }}"
+          else
+            echo "Using PR title as changelog: ${{ steps.job-vars.outputs.pr_title }}"
+          fi
+
       - name: Run airbyte-ci connectors --modified bump-version
         uses: ./.github/actions/run-airbyte-ci
         continue-on-error: true
@@ -86,10 +95,11 @@ jobs:
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          git_repo_url: https://github.com/${{ steps.job-vars.outputs.repo }}.git
           subcommand: |
             connectors --modified bump-version \
               ${{ github.event.inputs.type }} \
-              "${{ github.event.inputs.changelog }}" \
+              "${{ github.event.inputs.changelog != '' && github.event.inputs.changelog || steps.job-vars.outputs.pr_title }}" \
               --pr-number ${{ github.event.inputs.pr }}
 
       # This is helpful in the case that we change a previously committed generated file to be ignored by git.

.github/workflows/gradle.yml

@@ -51,7 +51,7 @@ jobs:
     # which explains why which we went down from 64 cores to 16.
     runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
     name: Gradle Check
-    timeout-minutes: 30
+    timeout-minutes: 60
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v4

.github/workflows/live_tests.yml

@@ -33,6 +33,10 @@ on:
         description: Use the local CDK when building the target connector
         default: "false"
         type: boolean
+      disable_proxy:
+        description: Disable proxy for requests
+        default: "false"
+        type: boolean
       connection_subset:
         description: The subset of connections to select from.
         required: true
@@ -108,6 +112,15 @@ jobs:
             echo "READ_WITH_STATE_FLAG=" >> $GITHUB_ENV
           fi
 
+      - name: Setup Proxy Flag
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if ${{ github.event.inputs.disable_proxy }}; then
+            echo "DISABLE_PROXY_FLAG=--connector_live_tests.disable-proxy" >> $GITHUB_ENV
+          else
+            echo "DISABLE_PROXY_FLAG=" >> $GITHUB_ENV
+          fi
+
       - name: Setup Connection Subset Option
         if: github.event_name == 'workflow_dispatch'
         run: |
@@ -129,4 +142,4 @@ jobs:
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=live --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}
+          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=live --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.DISABLE_PROXY_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}

.github/workflows/regression_tests.yml

@@ -33,6 +33,10 @@ on:
         description: Use the local CDK when building the target connector
         default: "false"
         type: boolean
+      disable_proxy:
+        description: Disable proxy for requests
+        default: "false"
+        type: boolean
       connection_subset:
         description: The subset of connections to select from.
         required: true
@@ -114,6 +118,15 @@ jobs:
             echo "READ_WITH_STATE_FLAG=" >> $GITHUB_ENV
           fi
 
+      - name: Setup Proxy Flag
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if ${{ github.event.inputs.disable_proxy }}; then
+            echo "DISABLE_PROXY_FLAG=--connector_live_tests.disable-proxy" >> $GITHUB_ENV
+          else
+            echo "DISABLE_PROXY_FLAG=" >> $GITHUB_ENV
+          fi
+
       - name: Setup Connection Subset Option
         if: github.event_name == 'workflow_dispatch'
         run: |
@@ -135,4 +148,4 @@ jobs:
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }} --global-status-check-context="Regression Tests" --global-status-check-description='Running regression tests'
+          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.DISABLE_PROXY_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }} --global-status-check-context="Regression Tests" --global-status-check-description='Running regression tests'

airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt

@@ -9,6 +9,7 @@ import io.micronaut.context.ApplicationContext
 import io.micronaut.context.RuntimeBeanDefinition
 import io.micronaut.context.env.CommandLinePropertySource
 import io.micronaut.context.env.Environment
+import io.micronaut.context.env.MapPropertySource
 import io.micronaut.core.cli.CommandLine as MicronautCommandLine
 import java.nio.file.Path
 import kotlin.system.exitProcess
@@ -17,19 +18,35 @@ import picocli.CommandLine.Model.ArgGroupSpec
 import picocli.CommandLine.Model.OptionSpec
 import picocli.CommandLine.Model.UsageMessageSpec
 
+// A micronaut environment, so that connectors can override micronaut properties
+// without needing to copy an entire application.yaml out of the core CDK.
+// Intended usage is for connectors to create
+// `destination-foo/src/main/resources/application-connector.yaml` with the
+// specific properties to override.
+const val CONNECTOR_OVERRIDE_ENV = "connector"
+
 /** Source connector entry point. */
 class AirbyteSourceRunner(
     /** CLI args. */
     args: Array<out String>,
+    additionalMicronautEnvs: List<String> = emptyList(),
     /** Environment variables. */
     systemEnv: Map<String, String> = System.getenv(),
     /** Micronaut bean definition overrides, used only for tests. */
     vararg testBeanDefinitions: RuntimeBeanDefinition<*>,
-) : AirbyteConnectorRunner("source", args, systemEnv, testBeanDefinitions) {
+) :
+    AirbyteConnectorRunner(
+        "source",
+        args,
+        additionalMicronautEnvs,
+        systemEnv,
+        micronautProperties = emptyMap(),
+        testBeanDefinitions
+    ) {
     companion object {
         @JvmStatic
-        fun run(vararg args: String) {
-            AirbyteSourceRunner(args).run<AirbyteConnectorRunnable>()
+        fun run(vararg args: String, additionalMicronautEnvs: List<String> = emptyList()) {
+            AirbyteSourceRunner(args, additionalMicronautEnvs).run<AirbyteConnectorRunnable>()
         }
     }
 }
@@ -38,15 +55,25 @@ class AirbyteSourceRunner(
 class AirbyteDestinationRunner(
     /** CLI args. */
     args: Array<out String>,
+    additionalMicronautEnvs: List<String> = emptyList(),
     /** Environment variables. */
     systemEnv: Map<String, String> = System.getenv(),
+    micronautProperties: Map<String, String> = emptyMap(),
     /** Micronaut bean definition overrides, used only for tests. */
     vararg testBeanDefinitions: RuntimeBeanDefinition<*>,
-) : AirbyteConnectorRunner("destination", args, systemEnv, testBeanDefinitions) {
+) :
+    AirbyteConnectorRunner(
+        "destination",
+        args,
+        additionalMicronautEnvs,
+        systemEnv,
+        micronautProperties,
+        testBeanDefinitions
+    ) {
     companion object {
         @JvmStatic
-        fun run(vararg args: String) {
-            AirbyteDestinationRunner(args).run<AirbyteConnectorRunnable>()
+        fun run(vararg args: String, additionalMicronautEnvs: List<String> = emptyList()) {
+            AirbyteDestinationRunner(args, additionalMicronautEnvs).run<AirbyteConnectorRunnable>()
         }
     }
 }
@@ -58,7 +85,9 @@ class AirbyteDestinationRunner(
 sealed class AirbyteConnectorRunner(
     val connectorType: String,
     val args: Array<out String>,
+    additionalMicronautEnvs: List<String> = emptyList(),
     systemEnv: Map<String, String>,
+    val micronautProperties: Map<String, String> = emptyMap(),
     val testBeanDefinitions: Array<out RuntimeBeanDefinition<*>>,
 ) {
     val envs: Array<String> =
@@ -69,7 +98,8 @@ sealed class AirbyteConnectorRunner(
             // any junit calls. This doesn't work if we launch the connector from a different
             // thread, e.g. `Dispatchers.IO`. Force the test env if needed. Some tests launch the
             // connector from the IO context to avoid blocking themselves.
-            listOfNotNull(Environment.TEST.takeIf { testBeanDefinitions.isNotEmpty() })
+            listOfNotNull(Environment.TEST.takeIf { testBeanDefinitions.isNotEmpty() }) +
+            additionalMicronautEnvs
 
     inline fun <reified R : Runnable> run() {
         val picocliCommandLineFactory = PicocliCommandLineFactory(this)
@@ -80,13 +110,19 @@ sealed class AirbyteConnectorRunner(
                 picocliCommandLineFactory.commands.options().map { it.longestName() },
             )
         val commandLinePropertySource = CommandLinePropertySource(micronautCommandLine)
+        val additionalPropertiesSource =
+            MapPropertySource("additional_properties", micronautProperties)
         val ctx: ApplicationContext =
-            ApplicationContext.builder(R::class.java, *envs)
+        // note that we put the override envs last.
+        // This ensures that micronaut gives those environments precedence
+        // (because the last environment's application-XYZ.yaml wins).
+        ApplicationContext.builder(R::class.java, *envs, CONNECTOR_OVERRIDE_ENV)
                 .propertySources(
                     *listOfNotNull(
                             airbytePropertySource,
                             commandLinePropertySource,
                             MetadataYamlPropertySource(),
+                            additionalPropertiesSource,
                         )
                         .toTypedArray(),
                 )

airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/command/ConfigurationSpecificationSupplier.kt

@@ -4,10 +4,13 @@ package io.airbyte.cdk.command
 import com.fasterxml.jackson.databind.JsonNode
 import io.airbyte.cdk.ConfigErrorException
 import io.airbyte.cdk.util.Jsons
+import io.github.oshai.kotlinlogging.KotlinLogging
 import io.micronaut.context.annotation.Value
 import jakarta.inject.Singleton
 import java.util.function.Supplier
 
+private val logger = KotlinLogging.logger {}
+
 /**
  * Supplies a valid [T] configuration POJO instance, based on the `airbyte.connector.config`
  * Micronaut property values:
@@ -39,11 +42,12 @@ class ConfigurationSpecificationSupplier<T : ConfigurationSpecification>(
             try {
                 Jsons.writeValueAsString(micronautPropertiesFallback)
             } catch (e: Exception) {
-                throw ConfigErrorException("failed to serialize fallback instance for $javaClass")
+                throw ConfigErrorException("failed to serialize fallback instance for $javaClass", e)
             }
         }
         val json: String = jsonPropertyValue ?: jsonMicronautFallback
-        return ValidatedJsonUtils.parseOne(javaClass, json)
+        val retVal = ValidatedJsonUtils.parseOne(javaClass, json)
+        return retVal
     }
 }
 
@@ -57,3 +61,7 @@ class ConfigurationSpecificationSupplier<T : ConfigurationSpecification>(
  * Strictly speaking, its subclasses are not really POJOs anymore, but who cares.
  */
 abstract class ConfigurationSpecification
+
+abstract class DbConfigurationSpecification(): ConfigurationSpecification() {
+    abstract var databaseName: String
+}

airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/command/ConnectorCommandLinePropertySource.kt

@@ -53,7 +53,8 @@ private fun resolveValues(
             log.warn { "File '$jsonFile' not found for '$cliOptionKey'." }
             continue
         }
-        values["$prefix.json"] = jsonFile.readText().replace("$", "\${:$}")
+        val fileContent  = jsonFile.readText().replace("$", "\${:$}")
+        values["$prefix.json"] = fileContent
     }
     return values
 }