From 27633c7420431720760c1b00e4ab6ec686ab41ce Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sun, 5 Feb 2023 11:37:21 -0800 Subject: [PATCH 1/5] do not rely on local cdk anymore --- airbyte-webapp/README.md | 9 +++++++++ airbyte-webapp/package.json | 2 +- .../scripts/load-declarative-schema.sh | 17 +++++++++++++++++ .../connector_manifest_openapi.yaml | 2 +- 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100755 airbyte-webapp/scripts/load-declarative-schema.sh diff --git a/airbyte-webapp/README.md b/airbyte-webapp/README.md index 39565a8e4fde..47a4ee17d4b9 100644 --- a/airbyte-webapp/README.md +++ b/airbyte-webapp/README.md @@ -26,6 +26,15 @@ Builds the app for production to the `build` folder.
Builds the app and Docker image and tags the image with `yourtag`. Note: needs to be run from the root directory of the Airbyte project. +### Using a custom version of the CDK declarative manifest schema for the connector builder UI + +When working on the connector builder UI and doing changes to the CDK and the webapp at the same time, you can start the dev server with `CDK_MANIFEST_PATH` or `CDK_VERSION` environment variables set to have the correct Typescript types built. If `CDK_VERSION` is set, it's loading the specified version of the CDK from pypi instead of the default one, if `CDK_MANIFEST_PATH` is set, it's copying the schema file locally. + +For example: +``` +CDK_MANIFEST_PATH=../../airbyte/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_componenpnt_schema.yaml npm start +``` + ## Entrypoints * `airbyte-webapp/src/App.tsx` is the entrypoint into the OSS version of the webapp. * `airbyte-webapp/src/packages/cloud/App.tsx` is the entrypoint into the Cloud version of the webapp. diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index ccd3589b49f8..09b5d1d75795 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -24,7 +24,7 @@ "stylelint": "stylelint 'src/**/*.{css,scss}'", "stylelint-check": "stylelint-config-prettier-scss-check", "license-check": "node ./scripts/license-check.js", - "generate-client": "orval", + "generate-client": "./scripts/load-declarative-schema.sh && orval", "validate-links": "ts-node --skip-project ./scripts/validate-links.ts" }, "dependencies": { diff --git a/airbyte-webapp/scripts/load-declarative-schema.sh b/airbyte-webapp/scripts/load-declarative-schema.sh new file mode 100755 index 000000000000..3a0cb69ccbf7 --- /dev/null +++ b/airbyte-webapp/scripts/load-declarative-schema.sh @@ -0,0 +1,17 @@ +set -e +mkdir -p build + +if [ -z "$CDK_VERSION" ] +then + version="0.25.0" +fi + + +if [ -z "$CDK_MANIFEST_PATH" ] +then + echo "Downloading CDK manifest $version from pypi" + curl -L https://pypi.python.org/packages/source/a/airbyte-cdk/airbyte-cdk-${version}.tar.gz | tar -xzO airbyte-cdk-${version}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml > build/declarative_component_schema.yaml +else + echo "Copying local CDK manifest version from $CDK_MANIFEST_PATH" + cp ${CDK_MANIFEST_PATH} build/declarative_component_schema.yaml +fi \ No newline at end of file diff --git a/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml index 690df8ae8fe4..cab17ecd5bee 100644 --- a/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml +++ b/airbyte-webapp/src/services/connectorBuilder/connector_manifest_openapi.yaml @@ -6,4 +6,4 @@ paths: {} components: schemas: ConnectorManifest: - $ref: "../../../../airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml" + $ref: "../../../build/declarative_component_schema.yaml" From 5e2bc9aef6c7d2e921d8f2c8c9a0df573de803c4 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sun, 5 Feb 2023 15:17:24 -0800 Subject: [PATCH 2/5] cache versioned schemas --- airbyte-webapp/scripts/load-declarative-schema.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/scripts/load-declarative-schema.sh b/airbyte-webapp/scripts/load-declarative-schema.sh index 3a0cb69ccbf7..f65d2d7aeca4 100755 --- a/airbyte-webapp/scripts/load-declarative-schema.sh +++ b/airbyte-webapp/scripts/load-declarative-schema.sh @@ -1,16 +1,25 @@ set -e mkdir -p build +# Make sure this is aligned with the CDK version of the connector builder server +DEFAULT_CDK_VERSION="0.26.0" + if [ -z "$CDK_VERSION" ] then - version="0.25.0" + CDK_VERSION=$DEFAULT_CDK_VERSION fi if [ -z "$CDK_MANIFEST_PATH" ] then - echo "Downloading CDK manifest $version from pypi" - curl -L https://pypi.python.org/packages/source/a/airbyte-cdk/airbyte-cdk-${version}.tar.gz | tar -xzO airbyte-cdk-${version}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml > build/declarative_component_schema.yaml + TARGET_FILE="build/declarative_component_schema-${CDK_VERSION}.yaml" + if [ ! -f "$TARGET_FILE" ]; then + echo "Downloading CDK manifest schema $CDK_VERSION from pypi" + curl -L https://pypi.python.org/packages/source/a/airbyte-cdk/airbyte-cdk-${CDK_VERSION}.tar.gz | tar -xzO airbyte-cdk-${CDK_VERSION}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml > ${TARGET_FILE} + else + echo "Found cached CDK manifest schema $CDK_VERSION" + fi + cp ${TARGET_FILE} build/declarative_component_schema.yaml else echo "Copying local CDK manifest version from $CDK_MANIFEST_PATH" cp ${CDK_MANIFEST_PATH} build/declarative_component_schema.yaml From 7b55ca79ee50a3cc256cc975cbf1022edd9280d5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sun, 5 Feb 2023 15:18:11 -0800 Subject: [PATCH 3/5] fix used version --- airbyte-webapp/scripts/load-declarative-schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/scripts/load-declarative-schema.sh b/airbyte-webapp/scripts/load-declarative-schema.sh index f65d2d7aeca4..3bf0cd3c15fb 100755 --- a/airbyte-webapp/scripts/load-declarative-schema.sh +++ b/airbyte-webapp/scripts/load-declarative-schema.sh @@ -2,7 +2,7 @@ set -e mkdir -p build # Make sure this is aligned with the CDK version of the connector builder server -DEFAULT_CDK_VERSION="0.26.0" +DEFAULT_CDK_VERSION="0.25.0" if [ -z "$CDK_VERSION" ] then From f4ac75d70ea565ac7a2bfcf8a080634bb506b2e8 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 6 Feb 2023 09:56:47 -0800 Subject: [PATCH 4/5] centralize used CDK version --- .github/workflows/gradle.yml | 2 ++ airbyte-connector-builder-server/CDK_VERSION | 1 + airbyte-connector-builder-server/setup.py | 4 +++- airbyte-webapp/scripts/load-declarative-schema.sh | 3 +-- 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 airbyte-connector-builder-server/CDK_VERSION diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index de819f3d81a7..0225a253dc04 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -85,6 +85,8 @@ jobs: - 'airbyte-db/**' frontend: - 'airbyte-api/src/main/openapi/config.yaml' + - 'airbyte-connector-builder-server/CDK_VERSION' + - 'airbyte-connector-builder-server/src/main/openapi/openapi.yaml' - 'airbyte-webapp/**' - 'airbyte-webapp-e2e-tests/**' diff --git a/airbyte-connector-builder-server/CDK_VERSION b/airbyte-connector-builder-server/CDK_VERSION new file mode 100644 index 000000000000..94a5fe438afc --- /dev/null +++ b/airbyte-connector-builder-server/CDK_VERSION @@ -0,0 +1 @@ +0.25.0 \ No newline at end of file diff --git a/airbyte-connector-builder-server/setup.py b/airbyte-connector-builder-server/setup.py index d9bd5a55db0c..5a1ee07f9c50 100644 --- a/airbyte-connector-builder-server/setup.py +++ b/airbyte-connector-builder-server/setup.py @@ -12,6 +12,8 @@ # The text of the README file README = (HERE / "README.md").read_text() +CDK_VERSION = (HERE / "CDK_VERSION").read_text() + setup( name="connector-builder-server", version="0.40.32", @@ -41,7 +43,7 @@ }, packages=find_packages(exclude=("unit_tests", "integration_tests", "docs")), package_data={}, - install_requires=["airbyte-cdk==0.25", "fastapi", "uvicorn"], + install_requires=[f"airbyte-cdk=={CDK_VERSION}", "fastapi", "uvicorn"], python_requires=">=3.9.11", extras_require={ "tests": [ diff --git a/airbyte-webapp/scripts/load-declarative-schema.sh b/airbyte-webapp/scripts/load-declarative-schema.sh index 3bf0cd3c15fb..9219fd8c2929 100755 --- a/airbyte-webapp/scripts/load-declarative-schema.sh +++ b/airbyte-webapp/scripts/load-declarative-schema.sh @@ -1,8 +1,7 @@ set -e mkdir -p build -# Make sure this is aligned with the CDK version of the connector builder server -DEFAULT_CDK_VERSION="0.25.0" +DEFAULT_CDK_VERSION=`cat ../../airbyte-connector-builder-server/CDK_VERSION` if [ -z "$CDK_VERSION" ] then From 8cd12a5f3630d7693efc3f9f9a57ef2f7f3c6512 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 6 Feb 2023 09:59:57 -0800 Subject: [PATCH 5/5] fix dpath --- airbyte-webapp/scripts/load-declarative-schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/scripts/load-declarative-schema.sh b/airbyte-webapp/scripts/load-declarative-schema.sh index 9219fd8c2929..c0e02781ddcd 100755 --- a/airbyte-webapp/scripts/load-declarative-schema.sh +++ b/airbyte-webapp/scripts/load-declarative-schema.sh @@ -1,7 +1,7 @@ set -e mkdir -p build -DEFAULT_CDK_VERSION=`cat ../../airbyte-connector-builder-server/CDK_VERSION` +DEFAULT_CDK_VERSION=`cat ../airbyte-connector-builder-server/CDK_VERSION` if [ -z "$CDK_VERSION" ] then