Sync cdk manifest between webapp and connector builder server

.github/workflows/gradle.yml

@@ -85,6 +85,8 @@ jobs:
               - 'airbyte-db/**'
             frontend:
               - 'airbyte-api/src/main/openapi/config.yaml'
+              - 'airbyte-connector-builder-server/CDK_VERSION'
+              - 'airbyte-connector-builder-server/src/main/openapi/openapi.yaml'
               - 'airbyte-webapp/**'
               - 'airbyte-webapp-e2e-tests/**'
 

airbyte-connector-builder-server/CDK_VERSION

@@ -0,0 +1 @@
+0.25.0

airbyte-connector-builder-server/setup.py

@@ -12,6 +12,8 @@
 # The text of the README file
 README = (HERE / "README.md").read_text()
 
+CDK_VERSION = (HERE / "CDK_VERSION").read_text()
+
 setup(
     name="connector-builder-server",
     version="0.40.32",
@@ -41,7 +43,7 @@
     },
     packages=find_packages(exclude=("unit_tests", "integration_tests", "docs")),
     package_data={},
-    install_requires=["airbyte-cdk==0.25", "fastapi", "uvicorn"],
+    install_requires=[f"airbyte-cdk=={CDK_VERSION}", "fastapi", "uvicorn"],
     python_requires=">=3.9.11",
     extras_require={
         "tests": [

airbyte-webapp/package.json

@@ -25,7 +25,7 @@
     "stylelint": "stylelint 'src/**/*.{css,scss}'",
     "stylelint-check": "stylelint-config-prettier-scss-check",
     "license-check": "node ./scripts/license-check.js",
-    "generate-client": "orval",
+    "generate-client": "./scripts/load-declarative-schema.sh && orval",
     "validate-links": "ts-node --skip-project ./scripts/validate-links.ts"
   },
   "dependencies": {

airbyte-webapp/scripts/load-declarative-schema.sh

@@ -0,0 +1,25 @@
+set -e
+mkdir -p build
+
+DEFAULT_CDK_VERSION=`cat ../airbyte-connector-builder-server/CDK_VERSION`
+
+if [ -z "$CDK_VERSION" ]
+then
+    CDK_VERSION=$DEFAULT_CDK_VERSION
+fi
+
+
+if [ -z "$CDK_MANIFEST_PATH" ]
+then
+    TARGET_FILE="build/declarative_component_schema-${CDK_VERSION}.yaml"
+    if [ ! -f "$TARGET_FILE" ]; then
+        echo "Downloading CDK manifest schema $CDK_VERSION from pypi"
+        curl -L https://pypi.python.org/packages/source/a/airbyte-cdk/airbyte-cdk-${CDK_VERSION}.tar.gz | tar -xzO airbyte-cdk-${CDK_VERSION}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml > ${TARGET_FILE}
+    else
+        echo "Found cached CDK manifest schema $CDK_VERSION"
+    fi
+    cp ${TARGET_FILE} build/declarative_component_schema.yaml
+else
+    echo "Copying local CDK manifest version from $CDK_MANIFEST_PATH"
+    cp ${CDK_MANIFEST_PATH} build/declarative_component_schema.yaml
+fi

airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml

@@ -6,4 +6,4 @@ paths: {}
 components:
   schemas:
     ConnectorManifest:
-      $ref: "../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml"
+      $ref: "../../../build/declarative_component_schema.yaml"

docs/contributing-to-airbyte/developing-locally.md

@@ -198,6 +198,15 @@ pnpm start
 
 - Happy Hacking!
 
+#### Using a custom version of the CDK declarative manifest schema for the connector builder UI
+
+When working on the connector builder UI and doing changes to the CDK and the webapp at the same time, you can start the dev server with `CDK_MANIFEST_PATH` or `CDK_VERSION` environment variables set to have the correct Typescript types built. If `CDK_VERSION` is set, it's loading the specified version of the CDK from pypi instead of the default one, if `CDK_MANIFEST_PATH` is set, it's copying the schema file locally.
+
+For example:
+```
+CDK_MANIFEST_PATH=../../airbyte/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_componenpnt_schema.yaml pnpm start
+```
+
 ### Connector Specification Caching
 
 The Configuration API caches connector specifications. This is done to avoid needing to run Docker everytime one is needed in the UI. Without this caching, the UI crawls. If you update the specification of a connector and need to clear this cache so the API / UI picks up the change, you have two options: