Airbyte CDK: move from gradle to poetry (#36250)

.github/workflows/publish-cdk-command-manually.yml

@@ -60,20 +60,25 @@ jobs:
   build-cdk:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-python@v4
+      - name: Install Python
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: "21"
+      - name: Install Poetry
+        id: install_poetry
+        uses: snok/install-poetry@v1
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
+      - name: Install Dependencies
+        id: install_dependencies
+        working-directory: airbyte-cdk/python
+        run: poetry install
       - name: Build CDK Package
-        run: (cd airbyte-cdk/python; ./gradlew --no-daemon --no-build-cache :build)
+        working-directory: airbyte-cdk/python
+        run: poetry run poe build
       - name: Post failure to Slack channel dev-connectors-extensibility
         if: ${{ failure() }}
         uses: slackapi/slack-github-action@v1.23.0
@@ -113,6 +118,9 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+      - name: Install Poetry
+        id: install_poetry
+        uses: snok/install-poetry@v1
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
@@ -122,16 +130,20 @@ jobs:
       - name: "Publish Python CDK: bump version"
         id: bumpversion
         run: |
-          pip install bumpversion
           cd airbyte-cdk/python
+          # Bump docker image version
+          # TODO: this should not be necessary, as it's the source-declarative-manifest one which should be published separately
+          pip install bumpversion
           bumpversion ${{ github.event.inputs.release-type }}
-          new_version="$(grep -i 'current_version = ' .bumpversion.cfg | sed -e 's/.* = //')"
+          # Bump package version
+          poetry version ${{ github.event.inputs.release-type }}
+          new_version="$(poetry version -s)"
           awk -v NEW_VERSION="$new_version" -v CHANGELOG_MESSAGE="${{ github.event.inputs.changelog-message }}" 'NR==3{print "## " NEW_VERSION "\n" CHANGELOG_MESSAGE "\n"}1' CHANGELOG.md > tmp && mv tmp CHANGELOG.md
           echo NEW_VERSION=$new_version >> $GITHUB_OUTPUT
       - name: Commit and Push Changes
         uses: stefanzweifel/git-auto-commit-action@v4
         with:
-          file_pattern: airbyte-cdk/python/setup.py airbyte-cdk/python/.bumpversion.cfg airbyte-cdk/python/CHANGELOG.md airbyte-cdk/python/Dockerfile
+          file_pattern: airbyte-cdk/python/pyproject.toml airbyte-cdk/python/.bumpversion.cfg airbyte-cdk/python/CHANGELOG.md airbyte-cdk/python/Dockerfile
           commit_message: 🤖 Bump ${{ github.event.inputs.release-type }} version of Python CDK
           commit_user_name: Octavia Squidington III
           commit_user_email: octavia-squidington-iii@users.noreply.github.com
@@ -165,6 +177,7 @@ jobs:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
 
   publish-cdk:
+    name: Publish Python CDK to PyPi
     needs: bump-version
     runs-on: ubuntu-latest
     steps:
@@ -173,16 +186,12 @@ jobs:
         with:
           repository: ${{ github.event.inputs.repo }}
           ref: ${{ github.event.inputs.gitref }}
-      - name: Publish Python Package
-        uses: mariamrf/py-package-publish-action@v1.1.0
+      - name: Build and publish to pypi
+        uses: JRubics/poetry-publish@v2.0
         with:
-          # specify the same version as in ~/.python-version
+          pypi_token: ${{ secrets.PYPI_TOKEN }}
           python_version: "3.10"
-          pip_version: "23.2"
-          subdir: "airbyte-cdk/python/"
-        env:
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+          package_directory: "airbyte-cdk/python"
       - name: Post failure to Slack channel dev-connectors-extensibility
         if: ${{ failure() }}
         uses: slackapi/slack-github-action@v1.23.0
@@ -224,11 +233,6 @@ jobs:
         with:
           repository: airbytehq/airbyte
           ref: ${{ github.event.inputs.gitref }}
-      - name: Install Java
-        uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: "21"
       - name: Install Python
         uses: actions/setup-python@v4
         with:

.github/workflows/python_cdk_tests.yml

@@ -43,16 +43,18 @@ jobs:
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: "21"
-      - name: Install Pip
-        run: curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3
-      - name: Install Pyenv
-        run: python3 -m pip install virtualenv --user
-      - name: Run Gradle Check on Python CDK
-        run: (cd airbyte-cdk/python && CI=true ./gradlew --no-daemon --no-build-cache --scan check)
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+      - name: Install Dependencies
+        id: install_dependencies
+        working-directory: airbyte-cdk/python
+        run: poetry install
+      - name: Build CDK Package
+        working-directory: airbyte-cdk/python
+        run: poetry run poe build
+      - name: Check Python CDK
+        working-directory: airbyte-cdk/python
+        run: poetry run poe check-ci
 
   set-instatus-incident-on-failure:
     name: Create Instatus Incident on Failure

.github/workflows/run-mypy-on-modified-cdk-files.yml

@@ -20,7 +20,7 @@ jobs:
       - run: pip install mypy==1.6.0
       - name: Get Python changed files
         id: changed-py-files
-        uses: tj-actions/changed-files@v23
+        uses: tj-actions/changed-files@v43
         with:
           files: "airbyte-cdk/python/airbyte_cdk/**/*.py"
       - name: Run if any of the listed files above is changed

airbyte-cdk/python/.bumpversion.cfg

@@ -1,7 +1,4 @@
 [bumpversion]
 current_version = 0.75.0
 commit = False
-
-[bumpversion:file:setup.py]
-
 [bumpversion:file:Dockerfile]

airbyte-cdk/python/README.md

@@ -9,18 +9,35 @@ The CDK provides an improved developer experience by providing basic implementat
 
 This document is a general introduction to the CDK. Readers should have basic familiarity with the [Airbyte Specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/) before proceeding.
 
-## Getting Started
+# Setup
 
-Generate an empty connector using the code generator. First clone the Airbyte repository then from the repository root run
+## Prerequisites
+
+#### Poetry
+
+Before you can start working on this project, you will need to have Poetry installed on your system. Please follow the instructions below to install Poetry:
+
+1. Open your terminal or command prompt.
+2. Install Poetry using the recommended installation method:
+
+```bash
+curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.5.1 python3 -
+```
+
+Alternatively, you can use `pip` to install Poetry:
 
 ```bash
-cd airbyte-integrations/connector-templates/generator
-./generate.sh
+pip install --user poetry
 ```
 
-then follow the interactive prompt. Next, find all `TODO`s in the generated project directory -- they're accompanied by lots of comments explaining what you'll need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector.
+3. After the installation is complete, close and reopen your terminal to ensure the newly installed `poetry` command is available in your system's PATH.
+
+For more detailed instructions and alternative installation methods, please refer to the official Poetry documentation: https://python-poetry.org/docs/#installation
+
+### Using Poetry in the Project
+
+Once Poetry is installed, you can use it to manage the project's dependencies and virtual environment. To get started, navigate to the project's root directory in your terminal and follow these steps:
 
-Additionally, you can follow [this tutorial](https://docs.airbyte.com/connector-development/cdk-python/) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK.
 
 ### Concepts & Documentation
 
@@ -42,39 +59,32 @@ See the [concepts docs](docs/concepts/) for a tour through what the API offers.
 
 ### First time setup
 
-We assume `python` points to Python 3.9 or higher.
-
-Setup a virtual env:
+Install the project dependencies and development tools:
 
 ```bash
-python -m venv .venv
-source .venv/bin/activate
-pip install -e ".[dev]" # [dev] installs development-only dependencies
+poetry install
 ```
 
 #### Iteration
 
-- Iterate on the code locally
-- Run tests via `python -m pytest -s unit_tests`
-- Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `mypy.ini`.
-- Run `mypy <files to check>` to only check specific files. This is useful as the CDK still contains code that is not compliant.
-- The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it!
+- Iterate on the CDK code locally
+- Run tests via `poetry run poe unit-test-with-cov`, or `python -m pytest -s unit_tests` if you want to pass pytest options.
+- Run `poetry run poe check` to lint and unit test the code in one command.
+- Run `bin/run-mypy-on-modified-files.sh` to run `mypy` static type checking on the files you have modified compared to master. This is useful as the CDK still contains code that is not compliant.
 
 ##### Autogenerated files
 
-If the iteration you are working on includes changes to the models, you might want to regenerate them. In order to do that, you can run:
+If the iteration you are working on includes changes to the models or the connector generator, you might want to regenerate them. In order to do that, you can run:
 
 ```bash
-cd airbyte-cdk/python
-./gradlew build
+poetry run poe build
 ```
 
-This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on
-pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateComponentManifestClassFiles`.
+This will generate the code generator docker image and the component manifest files based on the schemas and templates.
 
 #### Testing
 
-All tests are located in the `unit_tests` directory. Run `python -m pytest --cov=airbyte_cdk unit_tests/` to run them. This also presents a test coverage report.
+All tests are located in the `unit_tests` directory. Run `poetry run poe unit-test-with-cov` to run them. This also presents a test coverage report. For faster iteration with no coverage report and more options, `python -m pytest -s unit_tests` is a good place to start.
 
 #### Building and testing a connector with your local CDK
 
@@ -131,7 +141,7 @@ airbyte-ci connectors --use-local-cdk --name=<CONNECTOR> test
 
 #### When you don't have access to the API
 
-There can be some time where you do not have access to the API (either because you don't have the credentials, network access, etc...) You will probably still want to do end-to-end testing at least once. In order to do so, you can emulate the server you would be reaching using a server stubbing tool.
+There may be a time when you do not have access to the API (either because you don't have the credentials, network access, etc...) You will probably still want to do end-to-end testing at least once. In order to do so, you can emulate the server you would be reaching using a server stubbing tool.
 
 For example, using [mockserver](https://www.mock-server.com/), you can set up an expectation file like this:
 

airbyte-cdk/python/airbyte_cdk/connector_builder/README.md

@@ -30,11 +30,9 @@ Note:
 
 First, make sure you build the latest Docker image:
 ```
-./gradlew airbyte-cdk:python:airbyteDocker
+docker build -t airbyte/source-declarative-manifest:dev .
 ```
 
-The docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in the Dockerfile.
-
 #### Run
 
 Then run any of the connector commands as follows:

airbyte-cdk/python/bin/generate-component-manifest-files.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-[ -z "$ROOT_DIR" ] && exit 1
+ROOT_DIR=$(cd ../../ && pwd)
 
 YAML_DIR=airbyte-cdk/python/airbyte_cdk/sources/declarative
 OUTPUT_DIR=airbyte-cdk/python/airbyte_cdk/sources/declarative/models
@@ -33,7 +33,9 @@ function main() {
     # We can revisit this if there is movement on a fix.
     temp_file=$(mktemp)
     sed 's/ _parameters:/ parameters:/g' "$ROOT_DIR/$OUTPUT_DIR/$filename_wo_ext.py" > "${temp_file}"
-    mv "${temp_file}" "$ROOT_DIR/$OUTPUT_DIR/$filename_wo_ext.py"
+    output_file="$ROOT_DIR/$OUTPUT_DIR/$filename_wo_ext.py"
+    mv "${temp_file}" "${output_file}"
+    echo "Generated component manifest files into '${output_file}'."
   done
 }
 

airbyte-cdk/python/bin/run-mypy-on-modified-files.sh

@@ -7,7 +7,7 @@ cd "$(dirname "${0}")/.." || exit 1
 
 # TODO change this to include unit_tests as well once it's in a good state
 {
-  git diff --name-only --relative ':(exclude)unit_tests'
-  git diff --name-only --staged --relative ':(exclude)unit_tests'
-  git diff --name-only master... --relative ':(exclude)unit_tests'
-} | grep -E '\.py$' | sort | uniq | xargs .venv/bin/python -m mypy --config-file mypy.ini --install-types --non-interactive
+  git diff --name-only --diff-filter=d --relative ':(exclude)unit_tests'
+  git diff --name-only --diff-filter=d --staged --relative ':(exclude)unit_tests'
+  git diff --name-only --diff-filter=d master... --relative ':(exclude)unit_tests'
+} | grep -E '\.py$' | sort | uniq | xargs mypy --config-file mypy.ini --install-types --non-interactive