add the ability to upgrade CDK for java connectors (#34343)

We already had the ability to update the CDK version for python connectors. I'm just adding the same thing for java connectors.

It required some minor refactoring. I haven't added any tests yet, but I tested it locally

airbyte-ci/connectors/pipelines/README.md

@@ -578,6 +578,7 @@ E.G.: running `pytest` on a specific test folder:
 
 | Version | PR                                                         | Description                                                                                                       |
 | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| 3.7.0   | [#34343](https://github.com/airbytehq/airbyte/pull/34343)  | allow running connector upgrade_cdk for java connectors                                                             |
 | 3.6.1   | [#34490](https://github.com/airbytehq/airbyte/pull/34490)  | Fix inconsistent dagger log path typing                                                                           |
 | 3.6.0   | [#34111](https://github.com/airbytehq/airbyte/pull/34111)  | Add python registry publishing                                                                                    |
 | 3.5.3   | [#34339](https://github.com/airbytehq/airbyte/pull/34339)  | only do minimal changes on a connector version_bump                                                               |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py

@@ -3,26 +3,14 @@
 #
 
 import asyncclick as click
-import requests  # type: ignore
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
 from pipelines.airbyte_ci.connectors.upgrade_cdk.pipeline import run_connector_cdk_upgrade_pipeline
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 
 
-def latest_cdk_version() -> str:
-    """
-    Get the latest version of airbyte-cdk from pypi
-    """
-    cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json"
-    response = requests.get(cdk_pypi_url)
-    response.raise_for_status()
-    package_info = response.json()
-    return package_info["info"]["version"]
-
-
 @click.command(cls=DaggerPipelineCommand, short_help="Upgrade CDK version")
-@click.argument("target-cdk-version", type=str, default=latest_cdk_version)
+@click.argument("target-cdk-version", type=str, default="latest")
 @click.pass_context
 async def bump_version(
     ctx: click.Context,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py

@@ -7,12 +7,17 @@
 import re
 from typing import TYPE_CHECKING
 
+from connector_ops.utils import ConnectorLanguage  # type: ignore
+from dagger import Directory
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report
 from pipelines.helpers import git
+from pipelines.helpers.connectors import cdk_helpers
 from pipelines.models.steps import Step, StepResult, StepStatus
 
 if TYPE_CHECKING:
+    from typing import Optional
+
     from anyio import Semaphore
 
 
@@ -30,14 +35,22 @@ def __init__(
 
     async def _run(self) -> StepResult:
         context = self.context
-        og_connector_dir = await context.get_connector_dir()
-        if "setup.py" not in await og_connector_dir.entries():
-            return self.skip("Connector does not have a setup.py file.")
-        setup_py = og_connector_dir.file("setup.py")
-        setup_py_content = await setup_py.contents()
+
         try:
-            updated_setup_py = self.update_cdk_version(setup_py_content)
-            updated_connector_dir = og_connector_dir.with_new_file("setup.py", updated_setup_py)
+            og_connector_dir = await context.get_connector_dir()
+            if self.context.connector.language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]:
+                updated_connector_dir = await self.upgrade_cdk_version_for_python_connector(og_connector_dir)
+            elif self.context.connector.language is ConnectorLanguage.JAVA:
+                updated_connector_dir = await self.upgrade_cdk_version_for_java_connector(og_connector_dir)
+            else:
+                return StepResult(
+                    self,
+                    StepStatus.FAILURE,
+                    stderr=f"No CDK for connector {self.context.connector.technical_name} of written in {self.context.connector.language}",
+                )
+
+            if updated_connector_dir is None:
+                return self.skip(self.skip_reason)
             diff = og_connector_dir.diff(updated_connector_dir)
             exported_successfully = await diff.export(os.path.join(git.get_git_repo_path(), context.connector.code_directory))
             if not exported_successfully:
@@ -55,17 +68,57 @@ async def _run(self) -> StepResult:
                 exc_info=e,
             )
 
-    def update_cdk_version(self, og_setup_py_content: str) -> str:
+    async def upgrade_cdk_version_for_java_connector(self, og_connector_dir: Directory) -> Directory:
+        if "build.gradle" not in await og_connector_dir.entries():
+            raise ValueError(f"Java connector {self.context.connector.technical_name} does not have a build.gradle file.")
+
+        build_gradle = og_connector_dir.file("build.gradle")
+        build_gradle_content = await build_gradle.contents()
+
+        old_cdk_version_required = re.search(r"cdkVersionRequired *= *'(?P<version>[0-9]*\.[0-9]*\.[0-9]*)?'", build_gradle_content)
+        # If there is no airbyte-cdk dependency, add the version
+        if old_cdk_version_required is None:
+            raise ValueError("Could not find airbyte-cdk dependency in build.gradle")
+
+        if self.new_version == "latest":
+            new_version = await cdk_helpers.get_latest_java_cdk_version(self.context.get_repo_dir())
+        else:
+            new_version = self.new_version
+
+        updated_build_gradle = build_gradle_content.replace(old_cdk_version_required.group("version"), new_version)
+
+        use_local_cdk = re.search(r"useLocalCdk *=.*", updated_build_gradle)
+        if use_local_cdk is not None:
+            updated_build_gradle = updated_build_gradle.replace(use_local_cdk.group(), "useLocalCdk = false")
+
+        return og_connector_dir.with_new_file("build.gradle", updated_build_gradle)
+
+    async def upgrade_cdk_version_for_python_connector(self, og_connector_dir: Directory) -> Optional[Directory]:
+        context = self.context
+        og_connector_dir = await context.get_connector_dir()
+        if "setup.py" not in await og_connector_dir.entries():
+            self.skip_reason = f"Python connector {self.context.connector.technical_name} does not have a setup.py file."
+            return None
+        setup_py = og_connector_dir.file("setup.py")
+        setup_py_content = await setup_py.contents()
+
         airbyte_cdk_dependency = re.search(
-            r"airbyte-cdk(?P<extra>\[[a-zA-Z0-9-]*\])?(?P<version>[<>=!~]+[0-9]*\.[0-9]*\.[0-9]*)?", og_setup_py_content
+            r"airbyte-cdk(?P<extra>\[[a-zA-Z0-9-]*\])?(?P<version>[<>=!~]+[0-9]*(?:\.[0-9]*)?(?:\.[0-9]*)?)?", setup_py_content
         )
         # If there is no airbyte-cdk dependency, add the version
-        if airbyte_cdk_dependency is not None:
-            new_version = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}>={self.new_version}"
-            return og_setup_py_content.replace(airbyte_cdk_dependency.group(), new_version)
-        else:
+        if airbyte_cdk_dependency is None:
             raise ValueError("Could not find airbyte-cdk dependency in setup.py")
 
+        if self.new_version == "latest":
+            new_version = cdk_helpers.get_latest_python_cdk_version()
+        else:
+            new_version = self.new_version
+
+        new_version_str = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}>={new_version}"
+        updated_setup_py = setup_py_content.replace(airbyte_cdk_dependency.group(), new_version_str)
+
+        return og_connector_dir.with_new_file("setup.py", updated_setup_py)
+
 
 async def run_connector_cdk_upgrade_pipeline(
     context: ConnectorContext,

airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/cdk_helpers.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+import re
+
+import requests  # type: ignore
+from dagger import Directory
+
+
+def get_latest_python_cdk_version() -> str:
+    """
+    Get the latest version of airbyte-cdk from pypi
+    """
+    cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json"
+    response = requests.get(cdk_pypi_url)
+    response.raise_for_status()
+    package_info = response.json()
+    return package_info["info"]["version"]
+
+
+async def get_latest_java_cdk_version(repo_dir: Directory) -> str:
+    version_file_content = await repo_dir.file("airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties").contents()
+    match = re.search(r"version *= *(?P<version>[0-9]*\.[0-9]*\.[0-9]*)", version_file_content)
+    if match:
+        return match.group("version")
+    raise ValueError("Could not find version in version.properties")

airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py

@@ -3,7 +3,7 @@
 from typing import Optional
 from urllib.parse import urlparse
 
-import requests
+import requests  # type: ignore
 
 
 def is_package_published(package_name: Optional[str], version: Optional[str], registry_url: str) -> bool:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.6.1"
+version = "3.7.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py

@@ -0,0 +1,133 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import json
+import random
+from pathlib import Path
+from typing import List
+from unittest.mock import AsyncMock, MagicMock
+
+import anyio
+import pytest
+from connector_ops.utils import Connector, ConnectorLanguage
+from dagger import Directory
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline
+from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline
+from pipelines.models.steps import StepStatus
+
+pytestmark = [
+    pytest.mark.anyio,
+]
+
+
+@pytest.fixture
+def sample_connector():
+    return Connector("source-postgres")
+
+
+def get_sample_build_gradle(airbyte_cdk_version: str, useLocalCdk: str):
+    return f"""import org.jsonschema2pojo.SourceType
+
+plugins {{
+    id 'application'
+    id 'airbyte-java-connector'
+    id "org.jsonschema2pojo" version "1.2.1"
+}}
+
+java {{
+    compileJava {{
+        options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked"
+    }}
+}}
+
+airbyteJavaConnector {{
+    cdkVersionRequired = '{airbyte_cdk_version}'
+    features = ['db-sources']
+    useLocalCdk = {useLocalCdk}
+}}
+
+
+application {{
+    mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource'
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+}}
+"""
+
+
+@pytest.fixture
+def connector_context(sample_connector, dagger_client, current_platform):
+    context = ConnectorContext(
+        pipeline_name="test",
+        connector=sample_connector,
+        git_branch="test",
+        git_revision="test",
+        report_output_prefix="test",
+        is_local=True,
+        use_remote_secrets=True,
+        targeted_platforms=[current_platform],
+    )
+    context.dagger_client = dagger_client
+    return context
+
+
+@pytest.mark.parametrize(
+    "build_gradle_content, expected_build_gradle_content",
+    [
+        (get_sample_build_gradle("1.2.3", "false"), get_sample_build_gradle("6.6.6", "false")),
+        (get_sample_build_gradle("1.2.3", "true"), get_sample_build_gradle("6.6.6", "false")),
+        (get_sample_build_gradle("6.6.6", "false"), get_sample_build_gradle("6.6.6", "false")),
+        (get_sample_build_gradle("6.6.6", "true"), get_sample_build_gradle("6.6.6", "false")),
+        (get_sample_build_gradle("7.0.0", "false"), get_sample_build_gradle("6.6.6", "false")),
+        (get_sample_build_gradle("7.0.0", "true"), get_sample_build_gradle("6.6.6", "false")),
+    ],
+)
+async def test_run_connector_cdk_upgrade_pipeline(
+    connector_context: ConnectorContext, build_gradle_content: str, expected_build_gradle_content: str
+):
+    full_og_connector_dir = await connector_context.get_connector_dir()
+    updated_connector_dir = full_og_connector_dir.with_new_file("build.gradle", build_gradle_content)
+
+    # For this test, replace the actual connector dir with an updated version that sets the build.gradle contents
+    connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir)
+
+    # Mock the diff method to record the resulting directory and return a mock to not actually export the diff to the repo
+    updated_connector_dir.diff = MagicMock(return_value=AsyncMock())
+    step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6")
+    step_result = await step.run()
+    assert step_result.status == StepStatus.SUCCESS
+
+    # Check that the resulting directory that got passed to the mocked diff method looks as expected
+    resulting_directory: Directory = await full_og_connector_dir.diff(updated_connector_dir.diff.call_args[0][0])
+    files = await resulting_directory.entries()
+    # validate only build.gradle is changed
+    assert files == ["build.gradle"]
+    build_gradle = resulting_directory.file("build.gradle")
+    actual_build_gradle_content = await build_gradle.contents()
+    assert expected_build_gradle_content == actual_build_gradle_content
+
+    # Assert that the diff was exported to the repo
+    assert updated_connector_dir.diff.return_value.export.call_count == 1
+
+
+async def test_skip_connector_cdk_upgrade_pipeline_on_missing_build_gradle(connector_context: ConnectorContext):
+    full_og_connector_dir = await connector_context.get_connector_dir()
+    updated_connector_dir = full_og_connector_dir.without_file("build.gradle")
+
+    connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir)
+
+    step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6")
+    step_result = await step.run()
+    assert step_result.status == StepStatus.FAILURE
+
+
+async def test_fail_connector_cdk_upgrade_pipeline_on_missing_airbyte_cdk(connector_context: ConnectorContext):
+    full_og_connector_dir = await connector_context.get_connector_dir()
+    updated_connector_dir = full_og_connector_dir.with_new_file("build.gradle", get_sample_build_gradle("abc", "false"))
+
+    connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir)
+
+    step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6")
+    step_result = await step.run()
+    assert step_result.status == StepStatus.FAILURE

airbyte-ci/connectors/pipelines/tests/test_upgrade_cdk.py → airbyte-ci/connectors/pipelines/tests/test_upgrade_python_cdk.py

@@ -69,6 +69,9 @@ def connector_context(sample_connector, dagger_client, current_platform):
         (get_sample_setup_py("airbyte-cdk==1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")),
         (get_sample_setup_py("airbyte-cdk>=1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")),
         (get_sample_setup_py("airbyte-cdk[file-based]>=1.2.3"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")),
+        (get_sample_setup_py("airbyte-cdk==1.2"), get_sample_setup_py("airbyte-cdk>=6.6.6")),
+        (get_sample_setup_py("airbyte-cdk>=1.2"), get_sample_setup_py("airbyte-cdk>=6.6.6")),
+        (get_sample_setup_py("airbyte-cdk[file-based]>=1.2"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")),
     ],
 )
 async def test_run_connector_cdk_upgrade_pipeline(