diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 95f88196c047..ba35b375c2a3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.40.21 +current_version = 0.40.22 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? @@ -42,6 +42,8 @@ serialize = [bumpversion:file:charts/airbyte/Chart.yaml] +[bumpversion:file:charts/airbyte-connector-builder-server/Chart.yaml] + [bumpversion:file:charts/airbyte/README.md] [bumpversion:file:docs/operator-guides/upgrading-airbyte.md] diff --git a/.env b/.env index dfb96db7ca3f..3febcf1d03b3 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.40.21 +VERSION=0.40.22 # When using the airbyte-db via default docker image CONFIG_ROOT=/data @@ -61,6 +61,7 @@ CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server:80 WEBAPP_URL=http://localhost:8000/ # Although not present as an env var, required for webapp configuration. API_URL=/api/v1/ +CONNECTOR_BUILDER_API_URL=/connector-builder-api ### JOBS ### # Relevant to scaling. @@ -112,3 +113,5 @@ METRIC_CLIENT= OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317" USE_STREAM_CAPABLE_STATE=true +AUTO_DETECT_SCHEMA=false + diff --git a/.env.dev b/.env.dev index de61acefef9e..4f25503451fc 100644 --- a/.env.dev +++ b/.env.dev @@ -28,6 +28,7 @@ SYNC_JOB_MAX_ATTEMPTS=3 SYNC_JOB_MAX_TIMEOUT_DAYS=3 WORKERS_MICRONAUT_ENVIRONMENTS=control-plane CRON_MICRONAUT_ENVIRONMENTS=control-plane +AUTO_DETECT_SCHEMA=false # Sentry SENTRY_DSN="" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e5e57d6c8f68..4516f1b59cce 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,14 +9,28 @@ /airbyte-config/init/src/main/resources/icons/*.svg @airbytehq/design # CDK and SAT -/airbyte-cdk/ @airbytehq/api-connectors-dx -/airbyte-integrations/bases/source-acceptance-tests/ @airbytehq/api-connectors-dx -/airbyte-integrations/connector-templates/ @airbytehq/api-connectors-dx +/airbyte-cdk/ @airbytehq/connector-extensibility +/airbyte-integrations/bases/source-acceptance-tests/ @airbytehq/connector-extensibility +/airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility # Protocol related items /airbyte-protocol/ @airbytehq/protocol-reviewers /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers +# Airbyte Maintainers +airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @airbytehq/airbyte-maintainers +airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java @airbytehq/airbyte-maintainers +airbyte-commons/src/main/ @airbytehq/airbyte-maintainers +airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java @airbytehq/airbyte-maintainers +airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java @airbytehq/airbyte-maintainers +airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @airbytehq/airbyte-maintainers +airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/ @airbytehq/airbyte-maintainers +airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @airbytehq/airbyte-maintainers +airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java @airbytehq/airbyte-maintainers +airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @airbytehq/airbyte-maintainers +*build.gradle @airbytehq/airbyte-maintainers +airbyte-integrations/connectors/**/build.gradle # ignore gradle files for individual connectors. + # Normalization /airbyte-integrations/bases/base-normalization/ @airbytehq/normalization @@ -47,4 +61,4 @@ /airbyte-integrations/connectors/destination-postgres/ @airbytehq/jdbc-connectors /airbyte-integrations/connectors/destination-redshift/ @airbytehq/jdbc-connectors /airbyte-integrations/connectors/destination-rockset/ @airbytehq/jdbc-connectors -/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors \ No newline at end of file +/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors diff --git a/.github/workflows/gke-kube-test-command.yml b/.github/workflows/gke-kube-test-command.yml index 997241ed1156..a99b66fd5704 100644 --- a/.github/workflows/gke-kube-test-command.yml +++ b/.github/workflows/gke-kube-test-command.yml @@ -17,28 +17,10 @@ on: required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} start-gke-kube-acceptance-test-runner: timeout-minutes: 10 name: Start GKE Kube Acceptance Test EC2 Runner runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -48,13 +30,20 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} gke-kube-acceptance-test: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. needs: start-gke-kube-acceptance-test-runner # required to start the main job when the runner is ready @@ -146,7 +135,6 @@ jobs: needs: - start-gke-kube-acceptance-test-runner # required to get output from the start-runner job - gke-kube-acceptance-test # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest if: ${{ always() }} steps: @@ -156,10 +144,22 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index c90a714d48e5..bcbccb376ad1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -81,23 +81,6 @@ jobs: - 'airbyte-api/src/main/openapi/config.yaml' - 'airbyte-webapp/**' - 'airbyte-webapp-e2e-tests/**' - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} # Uncomment to debug. # changes-output: @@ -172,7 +155,6 @@ jobs: name: "Connectors Base: Start Build EC2 Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: | needs.changes.outputs.build == 'true' || needs.changes.outputs.connectors == 'true' || needs.changes.outputs.db == 'true' || (always() && github.ref == 'refs/heads/master') @@ -184,13 +166,20 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} build-connectors-base: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. @@ -299,7 +288,6 @@ jobs: needs: - start-connectors-base-build-runner # required to get output from the start-runner job - build-connectors-base # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -311,11 +299,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-connectors-base-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }} @@ -325,7 +322,6 @@ jobs: name: "Frontend: Start EC2 Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: | needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/master' @@ -338,18 +334,24 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} frontend-build: name: "Frontend: Build" needs: - start-frontend-runner - - find_valid_pat runs-on: ${{ needs.start-frontend-runner.outputs.label }} steps: - name: Checkout Airbyte @@ -459,7 +461,6 @@ jobs: - start-frontend-runner # required to get output from the start-runner job - frontend-test # required to wait when the e2e-test job is done - frontend-build # required to wait when then build job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -471,11 +472,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-frontend-runner.outputs.label }} ec2-instance-id: ${{ needs.start-frontend-runner.outputs.ec2-instance-id }} @@ -487,7 +497,6 @@ jobs: name: "Platform: Start Build EC2 Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -498,13 +507,20 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} platform-build: name: "Platform: Build" # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. @@ -646,7 +662,6 @@ jobs: needs: - start-platform-build-runner # required to get output from the start-runner job - platform-build # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -658,11 +673,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-platform-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} @@ -673,7 +697,6 @@ jobs: name: "Platform: Start Kube Acceptance Test Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -684,6 +707,13 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner @@ -692,7 +722,7 @@ jobs: ec2-image-id: ami-0c1a9bc22624339d8 aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} kube-acceptance-test: name: "Platform: Acceptance Tests (Kube)" # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. @@ -842,7 +872,6 @@ jobs: needs: - start-kube-acceptance-test-runner # required to get output from the start-runner job - kube-acceptance-test # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -854,11 +883,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }} @@ -870,7 +908,6 @@ jobs: name: "Platform: Start Helm Acceptance Test Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -880,7 +917,14 @@ jobs: ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner @@ -889,7 +933,7 @@ jobs: ec2-image-id: ami-0c1a9bc22624339d8 aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} helm-acceptance-test: name: "Platform: Acceptance Tests (Helm)" @@ -1030,6 +1074,16 @@ jobs: with: name: Kubernetes Logs path: /tmp/kubernetes_logs/* + + - name: Upload test results to BuildPulse for flaky test detection + if: "!cancelled()" # Run this step even when the tests fail. Skip if the workflow is cancelled. + uses: Workshop64/buildpulse-action@main + with: + account: 59758427 + repository: 283046497 + path: "/actions-runner/_work/airbyte/airbyte/*" + key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }} + secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }} # # In case of self-hosted EC2 errors, remove this block. @@ -1039,7 +1093,6 @@ jobs: needs: - start-helm-acceptance-test-runner # required to get output from the start-runner job - helm-acceptance-test # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -1051,11 +1104,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 14ca7de9d5ed..72f4f7e80125 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -30,29 +30,11 @@ on: default: "true" jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} ## Gradle Build # In case of self-hosted EC2 errors, remove this block. start-publish-image-runner-0: name: Start Build EC2 Runner 0 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -62,19 +44,25 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher start-publish-image-runner-1: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 1 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -84,19 +72,25 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher start-publish-image-runner-2: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 2 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -106,19 +100,25 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher start-publish-image-runner-3: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 3 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -128,19 +128,25 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher start-publish-image-runner-4: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 4 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -150,13 +156,20 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher preprocess-matrix: needs: start-publish-image-runner-0 @@ -392,7 +405,6 @@ jobs: - start-publish-image-runner-0 # required to get output from the start-runner job - preprocess-matrix - publish-image # required to wait when the main job is done - - find_valid_pat - add-helpful-info-to-git-comment runs-on: ubuntu-latest steps: @@ -402,11 +414,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-publish-image-runner-0.outputs.label }} ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} stop-publish-image-runner-multi: @@ -420,7 +441,6 @@ jobs: - start-publish-image-runner-4 - preprocess-matrix - publish-image # required to wait when the main job is done - - find_valid_pat strategy: fail-fast: false matrix: @@ -451,10 +471,19 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ matrix.ec2-instance.label }} ec2-instance-id: ${{ matrix.ec2-instance.id }} diff --git a/.github/workflows/publish-connector-command.yml b/.github/workflows/publish-connector-command.yml index cc695436f7ad..3fcc2daef8f0 100644 --- a/.github/workflows/publish-connector-command.yml +++ b/.github/workflows/publish-connector-command.yml @@ -26,30 +26,12 @@ on: required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} ## Gradle Build # In case of self-hosted EC2 errors, remove this block. # start-bump-build-test-connector-runner: # name: Start Build EC2 Runner # runs-on: ubuntu-latest -# needs: find_valid_pat # outputs: # label: ${{ steps.start-ec2-runner.outputs.label }} # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -59,13 +41,20 @@ jobs: # with: # repository: ${{ github.event.inputs.repo }} # ref: ${{ github.event.inputs.gitref }} +# - name: Check PAT rate limits +# run: | +# ./tools/bin/find_non_rate_limited_PAT \ +# ${{ secrets.AIRBYTEIO_PAT }} \ +# ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ +# ${{ secrets.SUPERTOPHER_PAT }} \ +# ${{ secrets.DAVINCHIA_PAT }} # - name: Start AWS Runner # id: start-ec2-runner # uses: ./.github/actions/start-aws-runner # with: # aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} -# github-token: ${{ needs.find_valid_pat.outputs.pat }} +# github-token: ${{ env.PAT }} # # 80 gb disk # ec2-image-id: ami-06cf12549e3d9c522 # bump-build-test-connector: @@ -192,7 +181,6 @@ jobs: # needs: # - start-bump-build-test-connector-runner # required to get output from the start-runner job # - bump-build-test-connector # required to wait when the main job is done -# - find_valid_pat # runs-on: ubuntu-latest # if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs # steps: @@ -202,10 +190,19 @@ jobs: # aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} # aws-region: us-east-2 +# - name: Checkout Airbyte +# uses: actions/checkout@v3 +# - name: Check PAT rate limits +# run: | +# ./tools/bin/find_non_rate_limited_PAT \ +# ${{ secrets.AIRBYTEIO_PAT }} \ +# ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ +# ${{ secrets.SUPERTOPHER_PAT }} \ +# ${{ secrets.DAVINCHIA_PAT }} # - name: Stop EC2 runner # uses: supertopher/ec2-github-runner@base64v1.0.10 # with: # mode: stop -# github-token: ${{ needs.find_valid_pat.outputs.pat }} +# github-token: ${{ env.PAT }} # label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} # ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish-helm-charts.yml b/.github/workflows/publish-helm-charts.yml index b6f1540fbace..3ad583619da3 100644 --- a/.github/workflows/publish-helm-charts.yml +++ b/.github/workflows/publish-helm-charts.yml @@ -70,7 +70,7 @@ jobs: - name: "Helm package" shell: bash run: | - declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics" "airbyte-cron") + declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics" "airbyte-cron" "airbyte-connector-builder-server") for val in ${StringArray[@]}; do cd ./airbyte/charts/${val} && helm dep update && cd $GITHUB_WORKSPACE sed -i -E 's/version: \"[0-9]+\.[0-9]+\.[0-9]+\"/version: \"${{ needs.generate-semantic-version.outputs.next-version }}\"/' ./airbyte/charts/${val}/Chart.yaml diff --git a/.github/workflows/publish-oss-for-cloud.yml b/.github/workflows/publish-oss-for-cloud.yml index 74fb5486c6b3..ad31e647fc10 100644 --- a/.github/workflows/publish-oss-for-cloud.yml +++ b/.github/workflows/publish-oss-for-cloud.yml @@ -9,41 +9,30 @@ on: description: "Publish artifacts for the following git ref (if unspecified, uses the latest commit for the current branch):" required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" + start-runner: + name: "Start Runner on AWS" timeout-minutes: 10 runs-on: ubuntu-latest outputs: - pat: ${{ steps.variables.outputs.pat }} + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 - name: Check PAT rate limits - id: variables run: | ./tools/bin/find_non_rate_limited_PAT \ ${{ secrets.AIRBYTEIO_PAT }} \ ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ ${{ secrets.SUPERTOPHER_PAT }} \ ${{ secrets.DAVINCHIA_PAT }} - start-runner: - name: "Start Runner on AWS" - needs: find_valid_pat - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} generate-tags: name: "Generate Dev and Master Tags" diff --git a/.github/workflows/release-airbyte-os.yml b/.github/workflows/release-airbyte-os.yml index 2165dc688f51..48e665168185 100644 --- a/.github/workflows/release-airbyte-os.yml +++ b/.github/workflows/release-airbyte-os.yml @@ -9,42 +9,31 @@ on: required: true default: "patch" jobs: - find_valid_pat: - name: "Find a PAT with room for actions" + # In case of self-hosted EC2 errors, remove this block. + start-release-airbyte-runner: + name: "Release Airbyte: Start EC2 Runner" timeout-minutes: 10 runs-on: ubuntu-latest outputs: - pat: ${{ steps.variables.outputs.pat }} + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 - name: Check PAT rate limits - id: variables run: | ./tools/bin/find_non_rate_limited_PAT \ ${{ secrets.AIRBYTEIO_PAT }} \ ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ ${{ secrets.SUPERTOPHER_PAT }} \ ${{ secrets.DAVINCHIA_PAT }} - # In case of self-hosted EC2 errors, remove this block. - start-release-airbyte-runner: - name: "Release Airbyte: Start EC2 Runner" - needs: find_valid_pat - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} releaseAirbyte: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. @@ -163,7 +152,6 @@ jobs: needs: - start-release-airbyte-runner # required to get output from the start-runner job - releaseAirbyte # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -173,10 +161,19 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-release-airbyte-runner.outputs.label }} ec2-instance-id: ${{ needs.start-release-airbyte-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 23b0ff27469a..96e4cafbfa78 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -21,8 +21,8 @@ on: required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" + uuid: + name: "Custom UUID of workflow run" timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -30,19 +30,9 @@ jobs: steps: - name: UUID ${{ github.event.inputs.uuid }} run: true - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} start-test-runner: name: Start Build EC2 Runner - needs: find_valid_pat + needs: uuid timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -54,13 +44,20 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} # 80 gb disk ec2-image-id: ami-06cf12549e3d9c522 integration-test: @@ -188,7 +185,7 @@ jobs: needs: - start-test-runner # required to get output from the start-runner job - integration-test # required to wait when the main job is done - - find_valid_pat + - uuid runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -198,10 +195,19 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index c420ebc92811..11e51f2b2d10 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -26,26 +26,8 @@ on: required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} start-test-runner: name: Start Build EC2 Runner - needs: find_valid_pat timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -57,13 +39,20 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} # 80 gb disk ec2-image-id: ami-06cf12549e3d9c522 performance-test: @@ -188,7 +177,6 @@ jobs: needs: - start-test-runner # required to get output from the start-runner job - performance-test # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -198,10 +186,19 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }} diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java index 89c08ff3a45c..7a2efa51b722 100644 --- a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java +++ b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java @@ -4,6 +4,7 @@ package io.airbyte.api.client; +import com.google.common.annotations.VisibleForTesting; import io.airbyte.api.client.generated.AttemptApi; import io.airbyte.api.client.generated.ConnectionApi; import io.airbyte.api.client.generated.DbMigrationApi; @@ -19,6 +20,10 @@ import io.airbyte.api.client.generated.StateApi; import io.airbyte.api.client.generated.WorkspaceApi; import io.airbyte.api.client.invoker.generated.ApiClient; +import java.util.Random; +import java.util.concurrent.Callable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class is meant to consolidate all our API endpoints into a fluent-ish client. Currently, all @@ -34,6 +39,13 @@ */ public class AirbyteApiClient { + private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteApiClient.class); + private static final Random RANDOM = new Random(); + + public static final int DEFAULT_MAX_RETRIES = 4; + public static final int DEFAULT_RETRY_INTERVAL_SECS = 10; + public static final int DEFAULT_FINAL_INTERVAL_SECS = 10 * 60; + private final ConnectionApi connectionApi; private final DestinationDefinitionApi destinationDefinitionApi; private final DestinationApi destinationApi; @@ -128,4 +140,67 @@ public StateApi getStateApi() { return stateApi; } + /** + * Default to 4 retries with a randomised 1 - 10 seconds interval between the first two retries and + * an 10-minute wait for the last retry. + */ + public static T retryWithJitter(final Callable call, final String desc) { + return retryWithJitter(call, desc, DEFAULT_RETRY_INTERVAL_SECS, DEFAULT_FINAL_INTERVAL_SECS, DEFAULT_MAX_RETRIES); + } + + /** + * Provides a simple retry wrapper for api calls. This retry behaviour is slightly different from + * generally available retries libraries - the last retry is able to wait an interval inconsistent + * with regular intervals/exponential backoff. + *

+ * Since the primary retries use case is long-running workflows, the benefit of waiting a couple of + * minutes as a last ditch effort to outlast networking disruption outweighs the cost of slightly + * longer jobs. + * + * @param call method to execute + * @param desc short readable explanation of why this method is executed + * @param jitterMaxIntervalSecs upper limit of the randomised retry interval. Minimum value is 1. + * @param finalIntervalSecs retry interval before the last retry. + */ + @VisibleForTesting + // This is okay since we are logging the stack trace, which PMD is not detecting. + @SuppressWarnings("PMD.PreserveStackTrace") + public static T retryWithJitter(final Callable call, + final String desc, + final int jitterMaxIntervalSecs, + final int finalIntervalSecs, + final int maxTries) { + int currRetries = 0; + boolean keepTrying = true; + + T data = null; + while (keepTrying && currRetries < maxTries) { + try { + LOGGER.info("Attempt {} to {}", currRetries, desc); + data = call.call(); + + keepTrying = false; + } catch (final Exception e) { + LOGGER.info("Attempt {} to {} error: {}", currRetries, desc, e); + currRetries++; + + // Sleep anywhere from 1 to jitterMaxIntervalSecs seconds. + final var backoffTimeSecs = Math.max(RANDOM.nextInt(jitterMaxIntervalSecs + 1), 1); + var backoffTimeMs = backoffTimeSecs * 1000; + + if (currRetries == maxTries - 1) { + // sleep for finalIntervalMins on the last attempt. + backoffTimeMs = finalIntervalSecs * 1000; + } + + try { + Thread.sleep(backoffTimeMs); + } catch (final InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } + return data; + } + } diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index d1745349070b..cf605d97da51 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -3371,6 +3371,8 @@ components: format: uuid geography: $ref: "#/components/schemas/Geography" + nonBreakingChangesPreference: + $ref: "#/components/schemas/NonBreakingChangesPreference" ConnectionStateCreateOrUpdate: type: object required: diff --git a/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java b/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java new file mode 100644 index 000000000000..2c96d9ef264c --- /dev/null +++ b/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.api.client; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.concurrent.Callable; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; + +public class AirbyteApiClientTest { + + // These set of configurations are so each test case takes ~3 secs. + private static final int TEST_JITTER_INTERVAL_SECS = 1; + private static final int TEST_FINAL_INTERVAL_SECS = 1; + private static final int TEST_MAX_RETRIES = 2; + @Mock + private Callable mockCallable; + + @Nested + class RetryWithJitter { + + @Test + @DisplayName("Should not retry on success") + void ifSucceedShouldNotRetry() throws Exception { + mockCallable = mock(Callable.class); + when(mockCallable.call()).thenReturn("Success!"); + + AirbyteApiClient.retryWithJitter(mockCallable, "test", TEST_JITTER_INTERVAL_SECS, TEST_FINAL_INTERVAL_SECS, TEST_MAX_RETRIES); + + verify(mockCallable, times(1)).call(); + } + + @Test + @DisplayName("Should retry up to the configured max retries on continued errors") + void onlyRetryTillMaxRetries() throws Exception { + mockCallable = mock(Callable.class); + when(mockCallable.call()).thenThrow(new RuntimeException("Bomb!")); + + AirbyteApiClient.retryWithJitter(mockCallable, "test", TEST_JITTER_INTERVAL_SECS, TEST_FINAL_INTERVAL_SECS, TEST_MAX_RETRIES); + + verify(mockCallable, times(TEST_MAX_RETRIES)).call(); + + } + + @Test + @DisplayName("Should retry only if there are errors") + void onlyRetryOnErrors() throws Exception { + mockCallable = mock(Callable.class); + // Because we succeed on the second try, we should only call the method twice. + when(mockCallable.call()) + .thenThrow(new RuntimeException("Bomb!")) + .thenReturn("Success!"); + + AirbyteApiClient.retryWithJitter(mockCallable, "test", TEST_JITTER_INTERVAL_SECS, TEST_FINAL_INTERVAL_SECS, 3); + + verify(mockCallable, times(2)).call(); + + } + + } + +} diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index ebf9599e150b..fc12ed33fddd 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -1,7 +1,7 @@ ARG JDK_IMAGE=airbyte/airbyte-base-java-image:1.0 FROM ${JDK_IMAGE} -ARG VERSION=0.40.21 +ARG VERSION=0.40.22 ENV APPLICATION airbyte-bootloader ENV VERSION ${VERSION} diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index b37ad9f9520c..2d011511854f 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.10.0 +Low-code: Add `start_from_page` option to a PageIncrement class + ## 0.9.5 Low-code: Add jinja macro `format_datetime` diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 190442f88469..d81431886d11 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -109,7 +109,11 @@ class Config: extra = Extra.allow name: str = Field(..., description="The name of the stream") - type: EstimateType = Field(..., description="The type of estimate", title="estimate type") + type: EstimateType = Field( + ..., + description="Estimates are either per-stream (STREAM) or for the entire sync (SYNC). STREAM is preferred, and requires the source to count how many records are about to be emitted per-stream (e.g. there will be 100 rows from this table emitted). For the rare source which cannot tell which stream a record belongs to before reading (e.g. CDC databases), SYNC estimates can be emitted. Sources should not emit both STREAM and SOURCE estimates within a sync.\n", + title="estimate type", + ) namespace: Optional[str] = Field(None, description="The namespace of the stream") row_estimate: Optional[int] = Field( None, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json index 8076c6f4ad7f..4d3341b8e2ea 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/config_component_schema.json @@ -1102,6 +1102,9 @@ "properties": { "page_size": { "type": "integer" + }, + "start_from_page": { + "type": "integer" } } } diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index 530c8fbcfcc7..7f3ffaf207b5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -17,13 +17,15 @@ class PageIncrement(PaginationStrategy, JsonSchemaMixin): Attributes: page_size (int): the number of records to request + start_from_page (int): number of the initial page """ page_size: int options: InitVar[Mapping[str, Any]] + start_from_page: int = 0 def __post_init__(self, options: Mapping[str, Any]): - self._page = 0 + self._page = self.start_from_page def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: if len(last_records) < self.page_size: @@ -33,7 +35,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin return self._page def reset(self): - self._page = 0 + self._page = self.start_from_page def get_page_size(self) -> Optional[int]: return self.page_size diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 11222c1020ab..133323cb4827 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.9.5", + version="0.10.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index 9d85cf8298b9..699e0f794a50 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -10,15 +10,17 @@ @pytest.mark.parametrize( - "test_name, page_size, expected_next_page_token, expected_offset", + "test_name, page_size, start_from, expected_next_page_token, expected_offset", [ - ("test_same_page_size", 2, 1, 1), - ("test_larger_page_size", 3, None, 0), + ("test_same_page_size_start_from_0", 2, 1, 2, 2), + ("test_larger_page_size_start_from_0", 3, 1, None, 1), + ("test_same_page_size_start_from_1", 2, 0, 1, 1), + ("test_larger_page_size_start_from_0", 3, 0, None, 0) ], ) -def test_page_increment_paginator_strategy(test_name, page_size, expected_next_page_token, expected_offset): - paginator_strategy = PageIncrement(page_size, options={}) - assert paginator_strategy._page == 0 +def test_page_increment_paginator_strategy(test_name, page_size, start_from, expected_next_page_token, expected_offset): + paginator_strategy = PageIncrement(page_size, options={}, start_from_page=start_from) + assert paginator_strategy._page == start_from response = requests.Response() @@ -32,4 +34,4 @@ def test_page_increment_paginator_strategy(test_name, page_size, expected_next_p assert expected_offset == paginator_strategy._page paginator_strategy.reset() - assert 0 == paginator_strategy._page + assert start_from == paginator_strategy._page diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/sync/OrchestratorConstants.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/sync/OrchestratorConstants.java index 7a7c7806d7d0..21a0ff14b645 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/sync/OrchestratorConstants.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/sync/OrchestratorConstants.java @@ -65,7 +65,8 @@ public class OrchestratorConstants { EnvConfigs.STATE_STORAGE_S3_ACCESS_KEY, EnvConfigs.STATE_STORAGE_S3_SECRET_ACCESS_KEY, EnvConfigs.STATE_STORAGE_S3_REGION, - EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE)) + EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, + EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA)) .build(); public static final String INIT_FILE_ENV_MAP = "envMap.json"; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java index d205894479c6..fff282e412fc 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java @@ -173,6 +173,7 @@ public static FailureReason dbtFailure(final Throwable t, final Long jobId, fina public static FailureReason unknownOriginFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { return genericFailure(t, jobId, attemptNumber) + .withFailureOrigin(FailureOrigin.UNKNOWN) .withExternalMessage("An unknown failure occurred"); } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java index f7d8caa4be69..a94c00927431 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/AirbyteMessageTracker.java @@ -8,6 +8,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.hash.HashFunction; @@ -19,6 +20,7 @@ import io.airbyte.config.State; import io.airbyte.protocol.models.AirbyteControlConnectorConfigMessage; import io.airbyte.protocol.models.AirbyteControlMessage; +import io.airbyte.protocol.models.AirbyteEstimateTraceMessage; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.AirbyteStateMessage; @@ -51,6 +53,7 @@ public class AirbyteMessageTracker implements MessageTracker { private final Map streamToRunningCount; private final HashFunction hashFunction; private final BiMap nameNamespacePairToIndex; + private final Map nameNamespacePairToStreamStats; private final Map streamToTotalBytesEmitted; private final Map streamToTotalRecordsEmitted; private final StateDeltaTracker stateDeltaTracker; @@ -60,6 +63,11 @@ public class AirbyteMessageTracker implements MessageTracker { private final StateAggregator stateAggregator; private final boolean logConnectorMessages = new EnvVariableFeatureFlags().logConnectorMessages(); + // These variables support SYNC level estimates and are meant for sources where stream level + // estimates are not possible e.g. CDC sources. + private Long totalRecordsEstimatedSync; + private Long totalBytesEstimatedSync; + private short nextStreamIndex; /** @@ -78,6 +86,11 @@ private enum ConnectorType { DESTINATION } + /** + * POJO for all per-stream stats. + */ + private record StreamStats(long estimatedBytes, long emittedBytes, long estimatedRecords, long emittedRecords) {} + public AirbyteMessageTracker() { this(new StateDeltaTracker(STATE_DELTA_TRACKER_MEMORY_LIMIT_BYTES), new DefaultStateAggregator(new EnvVariableFeatureFlags().useStreamCapableState()), @@ -93,6 +106,7 @@ protected AirbyteMessageTracker(final StateDeltaTracker stateDeltaTracker, this.streamToRunningCount = new HashMap<>(); this.nameNamespacePairToIndex = HashBiMap.create(); this.hashFunction = Hashing.murmur3_32_fixed(); + this.nameNamespacePairToStreamStats = new HashMap<>(); this.streamToTotalBytesEmitted = new HashMap<>(); this.streamToTotalRecordsEmitted = new HashMap<>(); this.stateDeltaTracker = stateDeltaTracker; @@ -252,7 +266,7 @@ private void handleEmittedOrchestratorConnectorConfig(final AirbyteControlConnec */ private void handleEmittedTrace(final AirbyteTraceMessage traceMessage, final ConnectorType connectorType) { switch (traceMessage.getType()) { - case ESTIMATE -> handleEmittedEstimateTrace(traceMessage, connectorType); + case ESTIMATE -> handleEmittedEstimateTrace(traceMessage.getEstimate()); case ERROR -> handleEmittedErrorTrace(traceMessage, connectorType); default -> log.warn("Invalid message type for trace message: {}", traceMessage); } @@ -266,8 +280,34 @@ private void handleEmittedErrorTrace(final AirbyteTraceMessage errorTraceMessage } } - @SuppressWarnings("PMD") // until method is implemented - private void handleEmittedEstimateTrace(final AirbyteTraceMessage estimateTraceMessage, final ConnectorType connectorType) { + /** + * There are several assumptions here: + *

+ * - Assume the estimate is a whole number and not a sum i.e. each estimate replaces the previous + * estimate. + *

+ * - Sources cannot emit both STREAM and SYNC estimates in a same sync. Error out if this happens. + */ + @SuppressWarnings("PMD.AvoidDuplicateLiterals") + private void handleEmittedEstimateTrace(final AirbyteEstimateTraceMessage estimate) { + switch (estimate.getType()) { + case STREAM -> { + Preconditions.checkArgument(totalBytesEstimatedSync == null, "STREAM and SYNC estimates should not be emitted in the same sync."); + Preconditions.checkArgument(totalRecordsEstimatedSync == null, "STREAM and SYNC estimates should not be emitted in the same sync."); + + log.debug("Saving stream estimates for namespace: {}, stream: {}", estimate.getNamespace(), estimate.getName()); + nameNamespacePairToStreamStats.put( + new AirbyteStreamNameNamespacePair(estimate.getName(), estimate.getNamespace()), + new StreamStats(estimate.getByteEstimate(), 0L, estimate.getRowEstimate(), 0L)); + } + case SYNC -> { + Preconditions.checkArgument(nameNamespacePairToStreamStats.isEmpty(), "STREAM and SYNC estimates should not be emitted in the same sync."); + + log.debug("Saving sync estimates"); + totalBytesEstimatedSync = estimate.getByteEstimate(); + totalRecordsEstimatedSync = estimate.getRowEstimate(); + } + } } @@ -368,6 +408,17 @@ public Map getStreamToEmittedRecords() { entry -> nameNamespacePairToIndex.inverse().get(entry.getKey()), Entry::getValue)); } + /** + * Swap out stream indices for stream names and return total records estimated by stream. + */ + @Override + public Map getStreamToEstimatedRecords() { + return nameNamespacePairToStreamStats.entrySet().stream().collect( + Collectors.toMap( + Entry::getKey, + entry -> entry.getValue().estimatedRecords())); + } + /** * Swap out stream indices for stream names and return total bytes emitted by stream. */ @@ -377,6 +428,17 @@ public Map getStreamToEmittedBytes() { entry -> nameNamespacePairToIndex.inverse().get(entry.getKey()), Entry::getValue)); } + /** + * Swap out stream indices for stream names and return total bytes estimated by stream. + */ + @Override + public Map getStreamToEstimatedBytes() { + return nameNamespacePairToStreamStats.entrySet().stream().collect( + Collectors.toMap( + Entry::getKey, + entry -> entry.getValue().estimatedBytes())); + } + /** * Compute sum of emitted record counts across all streams. */ @@ -385,6 +447,20 @@ public long getTotalRecordsEmitted() { return streamToTotalRecordsEmitted.values().stream().reduce(0L, Long::sum); } + /** + * Compute sum of estimated record counts across all streams. + */ + @Override + public long getTotalRecordsEstimated() { + if (!nameNamespacePairToStreamStats.isEmpty()) { + return nameNamespacePairToStreamStats.values().stream() + .map(e -> e.estimatedRecords) + .reduce(0L, Long::sum); + } + + return totalRecordsEstimatedSync; + } + /** * Compute sum of emitted bytes across all streams. */ @@ -393,6 +469,20 @@ public long getTotalBytesEmitted() { return streamToTotalBytesEmitted.values().stream().reduce(0L, Long::sum); } + /** + * Compute sum of estimated bytes across all streams. + */ + @Override + public long getTotalBytesEstimated() { + if (!nameNamespacePairToStreamStats.isEmpty()) { + return nameNamespacePairToStreamStats.values().stream() + .map(e -> e.estimatedBytes) + .reduce(0L, Long::sum); + } + + return totalBytesEstimatedSync; + } + /** * Compute sum of committed record counts across all streams. If the delta tracker has exceeded its * capacity, return empty because committed record counts cannot be reliably computed. diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/MessageTracker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/MessageTracker.java index 09507ec7a374..a2f31bf250d8 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/MessageTracker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/MessageTracker.java @@ -66,6 +66,14 @@ public interface MessageTracker { */ Map getStreamToEmittedRecords(); + /** + * Get the per-stream estimated record count provided by + * {@link io.airbyte.protocol.models.AirbyteEstimateTraceMessage}. + * + * @return returns a map of estimated record count by stream name. + */ + Map getStreamToEstimatedRecords(); + /** * Get the per-stream emitted byte count. This includes messages that were emitted by the source, * but never committed by the destination. @@ -74,6 +82,14 @@ public interface MessageTracker { */ Map getStreamToEmittedBytes(); + /** + * Get the per-stream estimated byte count provided by + * {@link io.airbyte.protocol.models.AirbyteEstimateTraceMessage}. + * + * @return returns a map of estimated bytes by stream name. + */ + Map getStreamToEstimatedBytes(); + /** * Get the overall emitted record count. This includes messages that were emitted by the source, but * never committed by the destination. @@ -82,6 +98,13 @@ public interface MessageTracker { */ long getTotalRecordsEmitted(); + /** + * Get the overall estimated record count. + * + * @return returns the total count of estimated records across all streams. + */ + long getTotalRecordsEstimated(); + /** * Get the overall emitted bytes. This includes messages that were emitted by the source, but never * committed by the destination. @@ -90,6 +113,13 @@ public interface MessageTracker { */ long getTotalBytesEmitted(); + /** + * Get the overall estimated bytes. + * + * @return returns the total count of estimated bytes across all streams. + */ + long getTotalBytesEstimated(); + /** * Get the overall committed record count. * diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java index 7f8542f9b116..5183e643f13a 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java @@ -212,7 +212,8 @@ private Map getWorkerMetadata() { WorkerEnvConstants.WORKER_CONNECTOR_IMAGE, imageName, WorkerEnvConstants.WORKER_JOB_ID, jobId, WorkerEnvConstants.WORKER_JOB_ATTEMPT, String.valueOf(attempt), - EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, String.valueOf(featureFlags.useStreamCapableState())); + EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, String.valueOf(featureFlags.useStreamCapableState()), + EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, String.valueOf(featureFlags.autoDetectSchema())); } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java index 2aede7159739..244e85303c8c 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java @@ -8,6 +8,7 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.AirbyteErrorTraceMessage; +import io.airbyte.protocol.models.AirbyteEstimateTraceMessage; import io.airbyte.protocol.models.AirbyteGlobalState; import io.airbyte.protocol.models.AirbyteLogMessage; import io.airbyte.protocol.models.AirbyteMessage; @@ -102,29 +103,60 @@ public static AirbyteStreamState createStreamState(final String streamName) { return new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(streamName)); } + public static AirbyteMessage createStreamEstimateMessage(final String name, final String namespace, final long byteEst, final long rowEst) { + return createEstimateMessage(AirbyteEstimateTraceMessage.Type.STREAM, name, namespace, byteEst, rowEst); + } + + public static AirbyteMessage createSyncEstimateMessage(final long byteEst, final long rowEst) { + return createEstimateMessage(AirbyteEstimateTraceMessage.Type.SYNC, null, null, byteEst, rowEst); + } + + public static AirbyteMessage createEstimateMessage(AirbyteEstimateTraceMessage.Type type, + final String name, + final String namespace, + final long byteEst, + final long rowEst) { + final var est = new AirbyteEstimateTraceMessage() + .withType(type) + .withByteEstimate(byteEst) + .withRowEstimate(rowEst); + + if (name != null) { + est.withName(name); + } + if (namespace != null) { + est.withNamespace(namespace); + } + + return new AirbyteMessage() + .withType(Type.TRACE) + .withTrace(new AirbyteTraceMessage().withType(AirbyteTraceMessage.Type.ESTIMATE) + .withEstimate(est)); + } + + public static AirbyteMessage createErrorMessage(final String message, final Double emittedAt) { + return new AirbyteMessage() + .withType(AirbyteMessage.Type.TRACE) + .withTrace(createErrorTraceMessage(message, emittedAt)); + } + public static AirbyteTraceMessage createErrorTraceMessage(final String message, final Double emittedAt) { - return new AirbyteTraceMessage() - .withType(io.airbyte.protocol.models.AirbyteTraceMessage.Type.ERROR) - .withEmittedAt(emittedAt) - .withError(new AirbyteErrorTraceMessage().withMessage(message)); + return createErrorTraceMessage(message, emittedAt, null); } public static AirbyteTraceMessage createErrorTraceMessage(final String message, final Double emittedAt, final AirbyteErrorTraceMessage.FailureType failureType) { - return new AirbyteTraceMessage() + final var msg = new AirbyteTraceMessage() .withType(io.airbyte.protocol.models.AirbyteTraceMessage.Type.ERROR) - .withEmittedAt(emittedAt) - .withError(new AirbyteErrorTraceMessage().withMessage(message).withFailureType(failureType)); - } + .withError(new AirbyteErrorTraceMessage().withMessage(message)) + .withEmittedAt(emittedAt); - public static AirbyteMessage createTraceMessage(final String message, final Double emittedAt) { - return new AirbyteMessage() - .withType(AirbyteMessage.Type.TRACE) - .withTrace(new AirbyteTraceMessage() - .withType(AirbyteTraceMessage.Type.ERROR) - .withEmittedAt(emittedAt) - .withError(new AirbyteErrorTraceMessage().withMessage(message))); + if (failureType != null) { + msg.getError().withFailureType(failureType); + } + + return msg; } } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java index 7b915ed4943c..e13f44edb6b4 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java @@ -307,7 +307,7 @@ void testReplicationRunnableWorkerFailure() throws Exception { @Test void testOnlyStateAndRecordMessagesDeliveredToDestination() throws Exception { final AirbyteMessage LOG_MESSAGE = AirbyteMessageUtils.createLogMessage(Level.INFO, "a log message"); - final AirbyteMessage TRACE_MESSAGE = AirbyteMessageUtils.createTraceMessage("a trace message", 123456.0); + final AirbyteMessage TRACE_MESSAGE = AirbyteMessageUtils.createErrorMessage("a trace message", 123456.0); when(mapper.mapMessage(LOG_MESSAGE)).thenReturn(LOG_MESSAGE); when(mapper.mapMessage(TRACE_MESSAGE)).thenReturn(TRACE_MESSAGE); when(source.isFinished()).thenReturn(false, false, false, false, true); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java index edbad7d065f8..263396f1e1eb 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java @@ -179,4 +179,14 @@ void testOrderedFailures() throws Exception { assertEquals(failureReasonList.get(0), TRACE_FAILURE_REASON); } + @Test + void testUnknownOriginFailure() { + final Throwable t = new RuntimeException(); + final Long jobId = 12345L; + final Integer attemptNumber = 1; + final FailureReason failureReason = FailureHelper.unknownOriginFailure(t, jobId, attemptNumber); + assertEquals(FailureOrigin.UNKNOWN, failureReason.getFailureOrigin()); + assertEquals("An unknown failure occurred", failureReason.getExternalMessage()); + } + } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java index 5123b299453c..aed444225cf9 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/AirbyteMessageTrackerTest.java @@ -5,6 +5,8 @@ package io.airbyte.workers.internal; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import io.airbyte.commons.json.Jsons; @@ -19,6 +21,8 @@ import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -28,9 +32,10 @@ @ExtendWith(MockitoExtension.class) class AirbyteMessageTrackerTest { - private static final String STREAM_1 = "stream1"; - private static final String STREAM_2 = "stream2"; - private static final String STREAM_3 = "stream3"; + private static final String NAMESPACE_1 = "avengers"; + private static final String STREAM_1 = "iron man"; + private static final String STREAM_2 = "black widow"; + private static final String STREAM_3 = "hulk"; private static final String INDUCED_EXCEPTION = "induced exception"; private AirbyteMessageTracker messageTracker; @@ -277,11 +282,11 @@ void testGetTotalRecordsCommitted_emptyWhenCommitStateHashThrowsException() thro } @Test - void testGetFirstDestinationAndSourceMessages() throws Exception { - final AirbyteMessage sourceMessage1 = AirbyteMessageUtils.createTraceMessage("source trace 1", Double.valueOf(123)); - final AirbyteMessage sourceMessage2 = AirbyteMessageUtils.createTraceMessage("source trace 2", Double.valueOf(124)); - final AirbyteMessage destMessage1 = AirbyteMessageUtils.createTraceMessage("dest trace 1", Double.valueOf(125)); - final AirbyteMessage destMessage2 = AirbyteMessageUtils.createTraceMessage("dest trace 2", Double.valueOf(126)); + void testGetFirstDestinationAndSourceMessages() { + final AirbyteMessage sourceMessage1 = AirbyteMessageUtils.createErrorMessage("source trace 1", 123.0); + final AirbyteMessage sourceMessage2 = AirbyteMessageUtils.createErrorMessage("source trace 2", 124.0); + final AirbyteMessage destMessage1 = AirbyteMessageUtils.createErrorMessage("dest trace 1", 125.0); + final AirbyteMessage destMessage2 = AirbyteMessageUtils.createErrorMessage("dest trace 2", 126.0); messageTracker.acceptFromSource(sourceMessage1); messageTracker.acceptFromSource(sourceMessage2); messageTracker.acceptFromDestination(destMessage1); @@ -292,39 +297,118 @@ void testGetFirstDestinationAndSourceMessages() throws Exception { } @Test - void testGetFirstDestinationAndSourceMessagesWithNulls() throws Exception { - assertEquals(messageTracker.getFirstDestinationErrorTraceMessage(), null); - assertEquals(messageTracker.getFirstSourceErrorTraceMessage(), null); + void testGetFirstDestinationAndSourceMessagesWithNulls() { + assertNull(messageTracker.getFirstDestinationErrorTraceMessage()); + assertNull(messageTracker.getFirstSourceErrorTraceMessage()); } @Test - void testErrorTraceMessageFailureWithMultipleTraceErrors() throws Exception { - final AirbyteMessage sourceMessage1 = AirbyteMessageUtils.createTraceMessage("source trace 1", Double.valueOf(123)); - final AirbyteMessage sourceMessage2 = AirbyteMessageUtils.createTraceMessage("source trace 2", Double.valueOf(124)); - final AirbyteMessage destMessage1 = AirbyteMessageUtils.createTraceMessage("dest trace 1", Double.valueOf(125)); - final AirbyteMessage destMessage2 = AirbyteMessageUtils.createTraceMessage("dest trace 2", Double.valueOf(126)); + void testErrorTraceMessageFailureWithMultipleTraceErrors() { + final AirbyteMessage sourceMessage1 = AirbyteMessageUtils.createErrorMessage("source trace 1", 123.0); + final AirbyteMessage sourceMessage2 = AirbyteMessageUtils.createErrorMessage("source trace 2", 124.0); + final AirbyteMessage destMessage1 = AirbyteMessageUtils.createErrorMessage("dest trace 1", 125.0); + final AirbyteMessage destMessage2 = AirbyteMessageUtils.createErrorMessage("dest trace 2", 126.0); messageTracker.acceptFromSource(sourceMessage1); messageTracker.acceptFromSource(sourceMessage2); messageTracker.acceptFromDestination(destMessage1); messageTracker.acceptFromDestination(destMessage2); final FailureReason failureReason = FailureHelper.sourceFailure(sourceMessage1.getTrace(), Long.valueOf(123), 1); - assertEquals(messageTracker.errorTraceMessageFailure(Long.valueOf(123), 1), + assertEquals(messageTracker.errorTraceMessageFailure(123L, 1), failureReason); } @Test - void testErrorTraceMessageFailureWithOneTraceError() throws Exception { - final AirbyteMessage destMessage = AirbyteMessageUtils.createTraceMessage("dest trace 1", Double.valueOf(125)); + void testErrorTraceMessageFailureWithOneTraceError() { + final AirbyteMessage destMessage = AirbyteMessageUtils.createErrorMessage("dest trace 1", 125.0); messageTracker.acceptFromDestination(destMessage); final FailureReason failureReason = FailureHelper.destinationFailure(destMessage.getTrace(), Long.valueOf(123), 1); - assertEquals(messageTracker.errorTraceMessageFailure(Long.valueOf(123), 1), failureReason); + assertEquals(messageTracker.errorTraceMessageFailure(123L, 1), failureReason); } @Test - void testErrorTraceMessageFailureWithNoTraceErrors() throws Exception { - assertEquals(messageTracker.errorTraceMessageFailure(Long.valueOf(123), 1), null); + void testErrorTraceMessageFailureWithNoTraceErrors() { + assertEquals(messageTracker.errorTraceMessageFailure(123L, 1), null); + } + + @Nested + class Estimates { + + // receiving an estimate for two streams should save + @Test + @DisplayName("when given stream estimates, should return correct per-stream estimates") + void streamShouldSaveAndReturnIndividualStreamCountsCorrectly() { + final var est1 = AirbyteMessageUtils.createStreamEstimateMessage(STREAM_1, NAMESPACE_1, 100L, 10L); + final var est2 = AirbyteMessageUtils.createStreamEstimateMessage(STREAM_2, NAMESPACE_1, 200L, 10L); + + messageTracker.acceptFromSource(est1); + messageTracker.acceptFromSource(est2); + + final var streamToEstBytes = messageTracker.getStreamToEstimatedBytes(); + final var expStreamToEstBytes = Map.of( + new AirbyteStreamNameNamespacePair(STREAM_1, NAMESPACE_1), 100L, + new AirbyteStreamNameNamespacePair(STREAM_2, NAMESPACE_1), 200L); + assertEquals(expStreamToEstBytes, streamToEstBytes); + + final var streamToEstRecs = messageTracker.getStreamToEstimatedRecords(); + final var expStreamToEstRecs = Map.of( + new AirbyteStreamNameNamespacePair(STREAM_1, NAMESPACE_1), 10L, + new AirbyteStreamNameNamespacePair(STREAM_2, NAMESPACE_1), 10L); + assertEquals(expStreamToEstRecs, streamToEstRecs); + } + + @Test + @DisplayName("when given stream estimates, should return correct total estimates") + void streamShouldSaveAndReturnTotalCountsCorrectly() { + final var est1 = AirbyteMessageUtils.createStreamEstimateMessage(STREAM_1, NAMESPACE_1, 100L, 10L); + final var est2 = AirbyteMessageUtils.createStreamEstimateMessage(STREAM_2, NAMESPACE_1, 200L, 10L); + + messageTracker.acceptFromSource(est1); + messageTracker.acceptFromSource(est2); + + final var totalEstBytes = messageTracker.getTotalBytesEstimated(); + assertEquals(300L, totalEstBytes); + + final var totalEstRecs = messageTracker.getTotalRecordsEstimated(); + assertEquals(20L, totalEstRecs); + } + + @Test + @DisplayName("should error when given both Stream and Sync estimates") + void shouldErrorOnBothStreamAndSyncEstimates() { + final var est1 = AirbyteMessageUtils.createStreamEstimateMessage(STREAM_1, NAMESPACE_1, 100L, 10L); + final var est2 = AirbyteMessageUtils.createSyncEstimateMessage(200L, 10L); + + messageTracker.acceptFromSource(est1); + assertThrows(IllegalArgumentException.class, () -> messageTracker.acceptFromSource(est2)); + } + + @Test + @DisplayName("when given sync estimates, should return correct total estimates") + void syncShouldSaveAndReturnTotalCountsCorrectly() { + final var est = AirbyteMessageUtils.createSyncEstimateMessage(200L, 10L); + messageTracker.acceptFromSource(est); + + final var totalEstBytes = messageTracker.getTotalBytesEstimated(); + assertEquals(200L, totalEstBytes); + + final var totalEstRecs = messageTracker.getTotalRecordsEstimated(); + assertEquals(10L, totalEstRecs); + } + + @Test + @DisplayName("when given sync estimates, should not return any per-stream estimates") + void syncShouldNotHaveStreamEstimates() { + final var est = AirbyteMessageUtils.createSyncEstimateMessage(200L, 10L); + messageTracker.acceptFromSource(est); + + final var streamToEstBytes = messageTracker.getStreamToEstimatedBytes(); + assertTrue(streamToEstBytes.isEmpty()); + final var streamToEstRecs = messageTracker.getStreamToEstimatedRecords(); + assertTrue(streamToEstRecs.isEmpty()); + } + } } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java index 03bf563792b3..3be0ab36723b 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/AirbyteIntegrationLauncherTest.java @@ -53,7 +53,8 @@ class AirbyteIntegrationLauncherTest { WorkerEnvConstants.WORKER_CONNECTOR_IMAGE, FAKE_IMAGE, WorkerEnvConstants.WORKER_JOB_ID, JOB_ID, WorkerEnvConstants.WORKER_JOB_ATTEMPT, String.valueOf(JOB_ATTEMPT), - EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, String.valueOf(new EnvVariableFeatureFlags().useStreamCapableState())); + EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, String.valueOf(new EnvVariableFeatureFlags().useStreamCapableState()), + EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, String.valueOf(new EnvVariableFeatureFlags().autoDetectSchema())); private WorkerConfigs workerConfigs; @Mock diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java b/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java index 81eb74b4d642..a9926b8ac6a6 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java @@ -11,6 +11,7 @@ public class EnvVariableFeatureFlags implements FeatureFlags { public static final String USE_STREAM_CAPABLE_STATE = "USE_STREAM_CAPABLE_STATE"; + public static final String AUTO_DETECT_SCHEMA = "AUTO_DETECT_SCHEMA"; public static final String LOG_CONNECTOR_MESSAGES = "LOG_CONNECTOR_MESSAGES"; public static final String NEED_STATE_VALIDATION = "NEED_STATE_VALIDATION"; @@ -31,6 +32,11 @@ public boolean useStreamCapableState() { return getEnvOrDefault(USE_STREAM_CAPABLE_STATE, false, Boolean::parseBoolean); } + @Override + public boolean autoDetectSchema() { + return getEnvOrDefault(AUTO_DETECT_SCHEMA, false, Boolean::parseBoolean); + } + @Override public boolean logConnectorMessages() { return getEnvOrDefault(LOG_CONNECTOR_MESSAGES, false, Boolean::parseBoolean); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java b/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java index eb3b765ceef8..b9132a8c274c 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java @@ -16,6 +16,8 @@ public interface FeatureFlags { boolean useStreamCapableState(); + boolean autoDetectSchema(); + boolean logConnectorMessages(); boolean needStateValidation(); diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index 35337c1b738f..8769119ddd27 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -687,6 +687,8 @@ public interface Configs { */ int getActivityNumberOfAttempt(); + boolean getAutoDetectSchema(); + enum TrackingStrategy { SEGMENT, LOGGING diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 676c06c9fe58..ef3d7b6750d6 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -202,6 +202,7 @@ public class EnvConfigs implements Configs { private static final long DEFAULT_MAX_SYNC_WORKERS = 5; private static final long DEFAULT_MAX_NOTIFY_WORKERS = 5; private static final String DEFAULT_NETWORK = "host"; + private static final String AUTO_DETECT_SCHEMA = "AUTO_DETECT_SCHEMA"; public static final Map> JOB_SHARED_ENVS = Map.of( AIRBYTE_VERSION, (instance) -> instance.getAirbyteVersion().serialize(), @@ -1050,6 +1051,11 @@ public int getWorkflowFailureRestartDelaySeconds() { return Integer.parseInt(getEnvOrDefault(WORKFLOW_FAILURE_RESTART_DELAY_SECONDS, String.valueOf(10 * 60))); } + @Override + public boolean getAutoDetectSchema() { + return getEnvOrDefault(AUTO_DETECT_SCHEMA, false); + } + @Override public int getActivityNumberOfAttempt() { return Integer.parseInt(getEnvOrDefault(ACTIVITY_MAX_ATTEMPT, "5")); diff --git a/airbyte-config/config-models/src/main/resources/types/FailureReason.yaml b/airbyte-config/config-models/src/main/resources/types/FailureReason.yaml index 72dced892a78..3d10988d4f01 100644 --- a/airbyte-config/config-models/src/main/resources/types/FailureReason.yaml +++ b/airbyte-config/config-models/src/main/resources/types/FailureReason.yaml @@ -18,6 +18,7 @@ properties: - normalization - dbt - airbyte_platform + - unknown failureType: description: Categorizes well known errors into types for programmatic handling. If not set, the type of error is not well known. type: string diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java index 5a3572a1c3cb..a34646ce4e93 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StandardSyncPersistence.java @@ -21,6 +21,7 @@ import io.airbyte.config.ConfigSchema; import io.airbyte.config.ConfigWithMetadata; import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.NonBreakingChangesPreference; import io.airbyte.config.helpers.ScheduleHelpers; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; @@ -195,6 +196,8 @@ private void writeStandardSync(final StandardSync standardSync, final DSLContext .set(CONNECTION.BREAKING_CHANGE, standardSync.getBreakingChange()) .set(CONNECTION.GEOGRAPHY, Enums.toEnum(standardSync.getGeography().value(), io.airbyte.db.instance.configs.jooq.generated.enums.GeographyType.class).orElseThrow()) + .set(CONNECTION.NON_BREAKING_CHANGE_PREFERENCE, standardSync.getNonBreakingChangesPreference().value()) + .set(CONNECTION.NOTIFY_SCHEMA_CHANGES, standardSync.getNotifySchemaChanges()) .where(CONNECTION.ID.eq(standardSync.getConnectionId())) .execute(); @@ -238,6 +241,9 @@ private void writeStandardSync(final StandardSync standardSync, final DSLContext .set(CONNECTION.GEOGRAPHY, Enums.toEnum(standardSync.getGeography().value(), io.airbyte.db.instance.configs.jooq.generated.enums.GeographyType.class).orElseThrow()) .set(CONNECTION.BREAKING_CHANGE, standardSync.getBreakingChange()) + .set(CONNECTION.NON_BREAKING_CHANGE_PREFERENCE, + standardSync.getNonBreakingChangesPreference() == null ? NonBreakingChangesPreference.IGNORE.value() + : standardSync.getNonBreakingChangesPreference().value()) .set(CONNECTION.CREATED_AT, timestamp) .set(CONNECTION.UPDATED_AT, timestamp) .execute(); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java new file mode 100644 index 000000000000..3f8a221fc7db --- /dev/null +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.init.DatabaseInitializationException; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; +import io.airbyte.test.utils.DatabaseConnectionHelper; +import java.io.IOException; +import java.sql.SQLException; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.PostgreSQLContainer; + +/** + * This class exists to abstract away the lifecycle of the test container database and the config + * database schema. This is ALL it intends to do. Any additional functionality belongs somewhere + * else. It is useful for test suites that need to interact directly with the database. + * + * This class sets up a test container database and runs the config database migrations against it + * to provide the most up-to-date schema. + * + * What this class is NOT designed to do: + *

    + *
  • test migration behavior, only should be used to test query behavior against the current + * schema.
    • + *
  • expose database details -- if you are attempting to expose container, dataSource, dslContext, + * something is wrong.
    • + *
  • add test fixtures or helpers--do NOT put "generic" resource helper methods (e.g. + * createTestSource())
    • + *
+ * + * This comment is emphatically worded, because it is tempting to add things to this class. It has + * already happened in 3 previous iterations, and each time it takes multiple engineering days to + * fix it. + * + * Usage: + *
    + *
  • Extend: Extend this class. By doing so, it will automatically create the test container db + * and run migrations against it at the start of the test suite (@BeforeAll).
    • + *
  • Use database: As part of the @BeforeAll the database field is set. This is the only field + * that the extending class can access. It's lifecycle is fully managed by this class.
    • + *
  • Reset schema: To reset the database in between tests, call truncateAllTables() as part + * of @BeforeEach. This is the only method that this class exposes externally. It is exposed in such + * a way, because most test suites need to declare their own @BeforeEach, so it is easier for them + * to simply call this method there, then trying to apply a more complex inheritance scheme.
    • + *
+ * + * Note: truncateAllTables() works by truncating each table in the db, if you add a new table, you + * will need to add it to that method for it work as expected. + */ +@SuppressWarnings({"PMD.MutableStaticState", "PMD.SignatureDeclareThrowsException"}) +class BaseConfigDatabaseTest { + + static Database database; + + // keep these private, do not expose outside this class! + private static PostgreSQLContainer container; + private static DataSource dataSource; + private static DSLContext dslContext; + + /** + * Create db test container, sets up java database resources, and runs migrations. Should not be + * called externally. It is not private because junit cannot access private methods. + * + * @throws DatabaseInitializationException - db fails to initialize + * @throws IOException - failure when interacting with db. + */ + @BeforeAll + static void dbSetup() throws DatabaseInitializationException, IOException { + createDbContainer(); + setDb(); + migrateDb(); + } + + /** + * Close all resources (container, data source, dsl context, database). Should not be called + * externally. It is not private because junit cannot access private methods. + * + * @throws Exception - exception while closing resources + */ + @AfterAll + static void dbDown() throws Exception { + dslContext.close(); + DataSourceFactory.close(dataSource); + container.close(); + } + + /** + * Truncates tables to reset them. Designed to be used in between tests. + * + * Note: NEW TABLES -- When a new table is added to the db, it will need to be added here. + * + * @throws SQLException - failure in truncate query. + */ + static void truncateAllTables() throws SQLException { + database.query(ctx -> ctx + .execute( + """ + TRUNCATE TABLE + actor, + actor_catalog, + actor_catalog_fetch_event, + actor_definition, + actor_definition_workspace_grant, + actor_oauth_parameter, + connection, + connection_operation, + operation, + state, + stream_reset, + workspace, + workspace_service_account + """)); + } + + private static void createDbContainer() { + container = new PostgreSQLContainer<>("postgres:13-alpine") + .withDatabaseName("airbyte") + .withUsername("docker") + .withPassword("docker"); + container.start(); + } + + private static void setDb() { + dataSource = DatabaseConnectionHelper.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + database = new Database(dslContext); + } + + private static void migrateDb() throws IOException, DatabaseInitializationException { + final Flyway flyway = FlywayFactory.create( + dataSource, + StreamResetPersistenceTest.class.getName(), + ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + new ConfigsDatabaseTestProvider(dslContext, flyway).create(true); + } + +} diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java deleted file mode 100644 index da381448620e..000000000000 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseDatabaseConfigPersistenceTest.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.persistence; - -import static org.jooq.impl.DSL.asterisk; -import static org.jooq.impl.DSL.count; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.config.ConfigSchema; -import io.airbyte.config.DestinationConnection; -import io.airbyte.config.Geography; -import io.airbyte.config.SourceConnection; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.config.StandardSourceDefinition; -import io.airbyte.config.StandardSourceDefinition.ReleaseStage; -import io.airbyte.config.StandardSourceDefinition.SourceType; -import io.airbyte.config.StandardWorkspace; -import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; -import io.airbyte.db.Database; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.UUID; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import javax.sql.DataSource; -import org.flywaydb.core.Flyway; -import org.jooq.DSLContext; -import org.jooq.Record1; -import org.jooq.Result; -import org.jooq.Table; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.PostgreSQLContainer; - -/** - * This class provides downstream tests with constants and helpers. - */ -@SuppressWarnings({"PMD.MutableStaticState", "PMD.SignatureDeclareThrowsException"}) -class BaseDatabaseConfigPersistenceTest { - - static PostgreSQLContainer container; - static Database database; - static DatabaseConfigPersistence configPersistence; - static StandardSyncPersistence standardSyncPersistence; - static JsonSecretsProcessor jsonSecretsProcessor; - static DataSource dataSource; - static DSLContext dslContext; - static Flyway flyway; - - protected static final StandardSourceDefinition SOURCE_GITHUB = new StandardSourceDefinition() - .withName("GitHub") - .withSourceDefinitionId(UUID.fromString("ef69ef6e-aa7f-4af1-a01d-ef775033524e")) - .withDockerRepository("airbyte/source-github") - .withDockerImageTag("0.2.3") - .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/github") - .withIcon("github.svg") - .withSourceType(SourceType.API) - .withProtocolVersion("0.2.0") - .withTombstone(false); - protected static final StandardSourceDefinition SOURCE_POSTGRES = new StandardSourceDefinition() - .withName("Postgres") - .withSourceDefinitionId(UUID.fromString("decd338e-5647-4c0b-adf4-da0e75f5a750")) - .withDockerRepository("airbyte/source-postgres") - .withDockerImageTag("0.3.11") - .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/postgres") - .withIcon("postgresql.svg") - .withSourceType(SourceType.DATABASE) - .withTombstone(false); - protected static final StandardSourceDefinition SOURCE_CUSTOM = new StandardSourceDefinition() - .withName("Custom") - .withSourceDefinitionId(UUID.fromString("baba338e-5647-4c0b-adf4-da0e75f5a750")) - .withDockerRepository("airbyte/cusom") - .withDockerImageTag("0.3.11") - .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/postgres") - .withIcon("postgresql.svg") - .withSourceType(SourceType.DATABASE) - .withCustom(true) - .withReleaseStage(ReleaseStage.CUSTOM) - .withTombstone(false); - protected static final StandardDestinationDefinition DESTINATION_SNOWFLAKE = new StandardDestinationDefinition() - .withName("Snowflake") - .withDestinationDefinitionId(UUID.fromString("424892c4-daac-4491-b35d-c6688ba547ba")) - .withDockerRepository("airbyte/destination-snowflake") - .withDockerImageTag("0.3.16") - .withDocumentationUrl("https://docs.airbyte.io/integrations/destinations/snowflake") - .withProtocolVersion("0.2.0") - .withTombstone(false); - protected static final StandardDestinationDefinition DESTINATION_S3 = new StandardDestinationDefinition() - .withName("S3") - .withDestinationDefinitionId(UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362")) - .withDockerRepository("airbyte/destination-s3") - .withDockerImageTag("0.1.12") - .withDocumentationUrl("https://docs.airbyte.io/integrations/destinations/s3") - .withProtocolVersion("0.2.0") - .withTombstone(false); - protected static final StandardDestinationDefinition DESTINATION_CUSTOM = new StandardDestinationDefinition() - .withName("Custom") - .withDestinationDefinitionId(UUID.fromString("baba338e-5647-4c0b-adf4-da0e75f5a750")) - .withDockerRepository("airbyte/cusom") - .withDockerImageTag("0.3.11") - .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/postgres") - .withIcon("postgresql.svg") - .withCustom(true) - .withReleaseStage(StandardDestinationDefinition.ReleaseStage.CUSTOM) - .withTombstone(false); - private static final String CANNOT_BE_NULL = "can not be null"; - - @BeforeAll - static void dbSetup() { - container = new PostgreSQLContainer<>("postgres:13-alpine") - .withDatabaseName("airbyte") - .withUsername("docker") - .withPassword("docker"); - container.start(); - jsonSecretsProcessor = mock(JsonSecretsProcessor.class); - } - - @AfterAll - static void dbDown() { - container.close(); - } - - static void truncateAllTables() throws SQLException { - database.query(ctx -> ctx - .execute( - "TRUNCATE TABLE workspace_service_account, state, actor_catalog, actor_catalog_fetch_event, connection_operation, connection, operation, actor_oauth_parameter, " - + "actor, actor_definition, actor_definition_workspace_grant, workspace, stream_reset")); - } - - void writeSource(final ConfigPersistence configPersistence, final StandardSourceDefinition source) throws Exception { - configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), source); - } - - void writeSourceWithSourceConnection(final ConfigPersistence configPersistence, final StandardSourceDefinition source) - throws Exception { - configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), source); - final UUID connectionId = UUID.randomUUID(); - final UUID workspaceId = UUID.randomUUID(); - final StandardWorkspace workspace = new StandardWorkspace() - .withWorkspaceId(workspaceId) - .withName(CANNOT_BE_NULL) - .withSlug(CANNOT_BE_NULL) - .withInitialSetupComplete(true) - .withDefaultGeography(Geography.AUTO); - configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, workspaceId.toString(), workspace); - - final SourceConnection sourceConnection = new SourceConnection() - .withSourceId(connectionId) - .withWorkspaceId(workspaceId) - .withName(CANNOT_BE_NULL) - .withSourceDefinitionId(source.getSourceDefinitionId()); - configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, connectionId.toString(), sourceConnection); - } - - void writeDestination(final ConfigPersistence configPersistence, final StandardDestinationDefinition destination) - throws Exception { - configPersistence.writeConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), destination); - } - - void writeDestinationWithDestinationConnection(final ConfigPersistence configPersistence, - final StandardDestinationDefinition destination) - throws Exception { - configPersistence.writeConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), destination); - final UUID connectionId = UUID.randomUUID(); - final UUID workspaceId = UUID.randomUUID(); - final StandardWorkspace workspace = new StandardWorkspace() - .withWorkspaceId(workspaceId) - .withName(CANNOT_BE_NULL) - .withSlug(CANNOT_BE_NULL) - .withInitialSetupComplete(true) - .withDefaultGeography(Geography.AUTO); - configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, workspaceId.toString(), workspace); - - final DestinationConnection destinationConnection = new DestinationConnection() - .withDestinationId(connectionId) - .withWorkspaceId(workspaceId) - .withName(CANNOT_BE_NULL) - .withDestinationDefinitionId(destination.getDestinationDefinitionId()); - configPersistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, connectionId.toString(), destinationConnection); - } - - void writeDestinations(final ConfigPersistence configPersistence, final List destinations) - throws Exception { - final Map destinationsByID = destinations.stream() - .collect(Collectors.toMap(destinationDefinition -> destinationDefinition.getDestinationDefinitionId().toString(), Function.identity())); - configPersistence.writeConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destinationsByID); - } - - void deleteDestination(final ConfigPersistence configPersistence, final StandardDestinationDefinition destination) - throws Exception { - configPersistence.deleteConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString()); - } - - protected Map> getMapWithSet(final Map> input) { - return input.entrySet().stream().collect(Collectors.toMap( - Entry::getKey, - e -> e.getValue().collect(Collectors.toSet()))); - } - - // assertEquals cannot correctly check the equality of two maps with stream values, - // so streams are converted to sets before being compared. - protected void assertSameConfigDump(final Map> expected, final Map> actual) { - assertEquals(getMapWithSet(expected), getMapWithSet(actual)); - } - - protected void assertRecordCount(final int expectedCount, final Table table) throws Exception { - final Result> recordCount = database.query(ctx -> ctx.select(count(asterisk())).from(table).fetch()); - assertEquals(expectedCount, recordCount.get(0).value1()); - } - - protected void assertHasSource(final StandardSourceDefinition source) throws Exception { - assertEquals(source, configPersistence - .getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), - StandardSourceDefinition.class)); - } - - protected void assertHasDestination(final StandardDestinationDefinition destination) throws Exception { - assertEquals(destination, configPersistence - .getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), - StandardDestinationDefinition.class)); - } - -} diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index 88fe070a2078..e941f3d28ea5 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -31,20 +31,12 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.persistence.ConfigRepository.DestinationAndDefinition; import io.airbyte.config.persistence.ConfigRepository.SourceAndDefinition; -import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.init.DatabaseInitializationException; -import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; -import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; -import io.airbyte.db.instance.development.DevDatabaseMigrator; -import io.airbyte.db.instance.development.MigrationDevHelper; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.sql.SQLException; @@ -57,53 +49,24 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import javax.sql.DataSource; -import org.flywaydb.core.Flyway; import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; @SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.NPathComplexity"}) -class ConfigRepositoryE2EReadWriteTest { +class ConfigRepositoryE2EReadWriteTest extends BaseConfigDatabaseTest { - private static PostgreSQLContainer container; - private DataSource dataSource; - private DSLContext dslContext; - private Database database; private ConfigRepository configRepository; private DatabaseConfigPersistence configPersistence; - private Flyway flyway; private final static String DOCKER_IMAGE_TAG = "1.2.0"; private final static String CONFIG_HASH = "ConfigHash"; - @BeforeAll - public static void dbSetup() { - container = new PostgreSQLContainer<>("postgres:13-alpine") - .withDatabaseName("airbyte") - .withUsername("docker") - .withPassword("docker"); - container.start(); - } - @BeforeEach void setup() throws IOException, JsonValidationException, SQLException, DatabaseInitializationException, InterruptedException { - dataSource = DatabaseConnectionHelper.createDataSource(container); - dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); - flyway = FlywayFactory.create(dataSource, ConfigRepositoryE2EReadWriteTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, - ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); - database = new ConfigsDatabaseTestProvider(dslContext, flyway).create(false); configPersistence = spy(new DatabaseConfigPersistence(database)); configRepository = spy(new ConfigRepository(configPersistence, database, new ActorDefinitionMigrator(new ExceptionWrappingDatabase(database)), new StandardSyncPersistence(database))); - final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, flyway); - final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); - MigrationDevHelper.runLastMigration(devDatabaseMigrator); for (final StandardWorkspace workspace : MockData.standardWorkspaces()) { configRepository.writeStandardWorkspaceNoSecrets(workspace); } @@ -136,11 +99,6 @@ void setup() throws IOException, JsonValidationException, SQLException, Database database.transaction(ctx -> ctx.truncate(ACTOR_DEFINITION_WORKSPACE_GRANT).execute()); } - @AfterAll - public static void dbDown() { - container.close(); - } - @Test void testWorkspaceCountConnections() throws IOException { diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java index 799ce1e4b4d3..4ba3a04cbbc2 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceE2EReadWriteTest.java @@ -25,47 +25,26 @@ import io.airbyte.config.StandardSyncState; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.WorkspaceServiceAccount; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.FlywayFactory; -import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; -import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; -import io.airbyte.db.instance.development.DevDatabaseMigrator; -import io.airbyte.db.instance.development.MigrationDevHelper; -import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.List; import java.util.Map; import org.apache.commons.lang3.NotImplementedException; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @SuppressWarnings({"PMD.SignatureDeclareThrowsException", "PMD.JUnitTestsShouldIncludeAssert", "PMD.DetachedTestCase"}) -class DatabaseConfigPersistenceE2EReadWriteTest extends BaseDatabaseConfigPersistenceTest { +class DatabaseConfigPersistenceE2EReadWriteTest extends BaseConfigDatabaseTest { + + private ConfigPersistence configPersistence; + private StandardSyncPersistence standardSyncPersistence; @BeforeEach void setup() throws Exception { - dataSource = DatabaseConnectionHelper.createDataSource(container); - dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); - flyway = FlywayFactory.create(dataSource, DatabaseConfigPersistenceE2EReadWriteTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, - ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); - database = new ConfigsDatabaseTestProvider(dslContext, flyway).create(false); configPersistence = spy(new DatabaseConfigPersistence(database)); standardSyncPersistence = new StandardSyncPersistence(database); - final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, flyway); - final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); - MigrationDevHelper.runLastMigration(devDatabaseMigrator); - truncateAllTables(); - } - @AfterEach - void tearDown() throws Exception { - dslContext.close(); - DataSourceFactory.close(dataSource); + truncateAllTables(); } @Test diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java index 177b79441871..481ca301d142 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java @@ -8,6 +8,8 @@ import static io.airbyte.config.ConfigSchema.STANDARD_SOURCE_DEFINITION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; import static org.assertj.core.api.Assertions.assertThat; +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.count; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -18,19 +20,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ConfigSchema; import io.airbyte.config.ConfigWithMetadata; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.Geography; +import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSourceDefinition.ReleaseStage; +import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.StandardWorkspace; import io.airbyte.config.persistence.DatabaseConfigPersistence.ConnectorInfo; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.FlywayFactory; -import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; -import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; -import io.airbyte.db.instance.development.DevDatabaseMigrator; -import io.airbyte.db.instance.development.MigrationDevHelper; -import io.airbyte.test.utils.DatabaseConnectionHelper; import java.time.Duration; import java.time.Instant; import java.time.LocalDate; @@ -38,10 +38,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.UUID; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterEach; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.Table; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -50,29 +55,75 @@ * methods. */ @SuppressWarnings({"PMD.SignatureDeclareThrowsException", "PMD.ShortVariable", "PMD.JUnitTestsShouldIncludeAssert"}) -class DatabaseConfigPersistenceTest extends BaseDatabaseConfigPersistenceTest { +class DatabaseConfigPersistenceTest extends BaseConfigDatabaseTest { + + public static final String DEFAULT_PROTOCOL_VERSION = "0.2.0"; + protected static final StandardSourceDefinition SOURCE_GITHUB = new StandardSourceDefinition() + .withName("GitHub") + .withSourceDefinitionId(UUID.fromString("ef69ef6e-aa7f-4af1-a01d-ef775033524e")) + .withDockerRepository("airbyte/source-github") + .withDockerImageTag("0.2.3") + .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/github") + .withIcon("github.svg") + .withSourceType(SourceType.API) + .withProtocolVersion(DEFAULT_PROTOCOL_VERSION) + .withTombstone(false); + protected static final StandardSourceDefinition SOURCE_POSTGRES = new StandardSourceDefinition() + .withName("Postgres") + .withSourceDefinitionId(UUID.fromString("decd338e-5647-4c0b-adf4-da0e75f5a750")) + .withDockerRepository("airbyte/source-postgres") + .withDockerImageTag("0.3.11") + .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/postgres") + .withIcon("postgresql.svg") + .withSourceType(SourceType.DATABASE) + .withTombstone(false); + protected static final StandardSourceDefinition SOURCE_CUSTOM = new StandardSourceDefinition() + .withName("Custom") + .withSourceDefinitionId(UUID.fromString("baba338e-5647-4c0b-adf4-da0e75f5a750")) + .withDockerRepository("airbyte/cusom") + .withDockerImageTag("0.3.11") + .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/postgres") + .withIcon("postgresql.svg") + .withSourceType(SourceType.DATABASE) + .withCustom(true) + .withReleaseStage(ReleaseStage.CUSTOM) + .withTombstone(false); + protected static final StandardDestinationDefinition DESTINATION_SNOWFLAKE = new StandardDestinationDefinition() + .withName("Snowflake") + .withDestinationDefinitionId(UUID.fromString("424892c4-daac-4491-b35d-c6688ba547ba")) + .withDockerRepository("airbyte/destination-snowflake") + .withDockerImageTag("0.3.16") + .withDocumentationUrl("https://docs.airbyte.io/integrations/destinations/snowflake") + .withProtocolVersion(DEFAULT_PROTOCOL_VERSION) + .withTombstone(false); + protected static final StandardDestinationDefinition DESTINATION_S3 = new StandardDestinationDefinition() + .withName("S3") + .withDestinationDefinitionId(UUID.fromString("4816b78f-1489-44c1-9060-4b19d5fa9362")) + .withDockerRepository("airbyte/destination-s3") + .withDockerImageTag("0.1.12") + .withDocumentationUrl("https://docs.airbyte.io/integrations/destinations/s3") + .withProtocolVersion(DEFAULT_PROTOCOL_VERSION) + .withTombstone(false); + protected static final StandardDestinationDefinition DESTINATION_CUSTOM = new StandardDestinationDefinition() + .withName("Custom") + .withDestinationDefinitionId(UUID.fromString("baba338e-5647-4c0b-adf4-da0e75f5a750")) + .withDockerRepository("airbyte/cusom") + .withDockerImageTag("0.3.11") + .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/postgres") + .withIcon("postgresql.svg") + .withCustom(true) + .withReleaseStage(StandardDestinationDefinition.ReleaseStage.CUSTOM) + .withTombstone(false); + private static final String CANNOT_BE_NULL = "can not be null"; + + private DatabaseConfigPersistence configPersistence; @BeforeEach public void setup() throws Exception { - dataSource = DatabaseConnectionHelper.createDataSource(container); - dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); - flyway = FlywayFactory.create(dataSource, DatabaseConfigPersistenceTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, - ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); - database = new ConfigsDatabaseTestProvider(dslContext, flyway).create(false); configPersistence = spy(new DatabaseConfigPersistence(database)); - final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, flyway); - final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); - MigrationDevHelper.runLastMigration(devDatabaseMigrator); truncateAllTables(); } - @AfterEach - void tearDown() throws Exception { - dslContext.close(); - DataSourceFactory.close(dataSource); - } - @Test void testMultiWriteAndGetConfig() throws Exception { writeDestinations(configPersistence, Lists.newArrayList(DESTINATION_S3, DESTINATION_SNOWFLAKE)); @@ -151,7 +202,7 @@ void testDeleteConfig() throws Exception { void testGetConnectorRepositoryToInfoMap() throws Exception { final String connectorRepository = "airbyte/duplicated-connector"; final String oldVersion = "0.1.10"; - final String newVersion = "0.2.0"; + final String newVersion = DEFAULT_PROTOCOL_VERSION; final StandardSourceDefinition source1 = new StandardSourceDefinition() .withSourceDefinitionId(UUID.randomUUID()) .withName("source-1") @@ -207,13 +258,13 @@ void testInsertConfigRecord() throws Exception { @Test void testHasNewVersion() { - assertTrue(DatabaseConfigPersistence.hasNewVersion("0.1.99", "0.2.0")); + assertTrue(DatabaseConfigPersistence.hasNewVersion("0.1.99", DEFAULT_PROTOCOL_VERSION)); assertFalse(DatabaseConfigPersistence.hasNewVersion("invalid_version", "0.1.2")); } @Test void testHasNewPatchVersion() { - assertFalse(DatabaseConfigPersistence.hasNewPatchVersion("0.1.99", "0.2.0")); + assertFalse(DatabaseConfigPersistence.hasNewPatchVersion("0.1.99", DEFAULT_PROTOCOL_VERSION)); assertFalse(DatabaseConfigPersistence.hasNewPatchVersion("invalid_version", "0.3.1")); assertTrue(DatabaseConfigPersistence.hasNewPatchVersion("0.1.0", "0.1.3")); } @@ -281,4 +332,97 @@ void filterCustomDestination() { assertThat(filteredSourceMap).containsOnlyKeys(nonCustomKey); } + void writeSource(final ConfigPersistence configPersistence, final StandardSourceDefinition source) throws Exception { + configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), source); + } + + void writeSourceWithSourceConnection(final ConfigPersistence configPersistence, final StandardSourceDefinition source) + throws Exception { + configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), source); + final UUID connectionId = UUID.randomUUID(); + final UUID workspaceId = UUID.randomUUID(); + final StandardWorkspace workspace = new StandardWorkspace() + .withWorkspaceId(workspaceId) + .withName(CANNOT_BE_NULL) + .withSlug(CANNOT_BE_NULL) + .withInitialSetupComplete(true) + .withDefaultGeography(Geography.AUTO); + configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, workspaceId.toString(), workspace); + + final SourceConnection sourceConnection = new SourceConnection() + .withSourceId(connectionId) + .withWorkspaceId(workspaceId) + .withName(CANNOT_BE_NULL) + .withSourceDefinitionId(source.getSourceDefinitionId()); + configPersistence.writeConfig(ConfigSchema.SOURCE_CONNECTION, connectionId.toString(), sourceConnection); + } + + void writeDestination(final ConfigPersistence configPersistence, final StandardDestinationDefinition destination) + throws Exception { + configPersistence.writeConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), destination); + } + + void writeDestinationWithDestinationConnection(final ConfigPersistence configPersistence, + final StandardDestinationDefinition destination) + throws Exception { + configPersistence.writeConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), destination); + final UUID connectionId = UUID.randomUUID(); + final UUID workspaceId = UUID.randomUUID(); + final StandardWorkspace workspace = new StandardWorkspace() + .withWorkspaceId(workspaceId) + .withName(CANNOT_BE_NULL) + .withSlug(CANNOT_BE_NULL) + .withInitialSetupComplete(true) + .withDefaultGeography(Geography.AUTO); + configPersistence.writeConfig(ConfigSchema.STANDARD_WORKSPACE, workspaceId.toString(), workspace); + + final DestinationConnection destinationConnection = new DestinationConnection() + .withDestinationId(connectionId) + .withWorkspaceId(workspaceId) + .withName(CANNOT_BE_NULL) + .withDestinationDefinitionId(destination.getDestinationDefinitionId()); + configPersistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, connectionId.toString(), destinationConnection); + } + + void writeDestinations(final ConfigPersistence configPersistence, final List destinations) + throws Exception { + final Map destinationsByID = destinations.stream() + .collect(Collectors.toMap(destinationDefinition -> destinationDefinition.getDestinationDefinitionId().toString(), Function.identity())); + configPersistence.writeConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destinationsByID); + } + + void deleteDestination(final ConfigPersistence configPersistence, final StandardDestinationDefinition destination) + throws Exception { + configPersistence.deleteConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString()); + } + + protected Map> getMapWithSet(final Map> input) { + return input.entrySet().stream().collect(Collectors.toMap( + Entry::getKey, + e -> e.getValue().collect(Collectors.toSet()))); + } + + // assertEquals cannot correctly check the equality of two maps with stream values, + // so streams are converted to sets before being compared. + protected void assertSameConfigDump(final Map> expected, final Map> actual) { + assertEquals(getMapWithSet(expected), getMapWithSet(actual)); + } + + protected void assertRecordCount(final int expectedCount, final Table table) throws Exception { + final Result> recordCount = database.query(ctx -> ctx.select(count(asterisk())).from(table).fetch()); + assertEquals(expectedCount, recordCount.get(0).value1()); + } + + protected void assertHasSource(final StandardSourceDefinition source) throws Exception { + assertEquals(source, configPersistence + .getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), + StandardSourceDefinition.class)); + } + + protected void assertHasDestination(final StandardDestinationDefinition destination) throws Exception { + assertEquals(destination, configPersistence + .getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), + StandardDestinationDefinition.class)); + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java index 4bea431aa5a0..e077d8e20630 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java @@ -5,32 +5,29 @@ package io.airbyte.config.persistence; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.count; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConfigSchema; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSourceDefinition.SourceType; import io.airbyte.config.persistence.DatabaseConfigPersistence.ConnectorInfo; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.FlywayFactory; -import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; -import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; -import io.airbyte.db.instance.development.DevDatabaseMigrator; -import io.airbyte.db.instance.development.MigrationDevHelper; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.test.utils.DatabaseConnectionHelper; import java.io.IOException; import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.Table; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -39,37 +36,31 @@ * Unit test for the {@link DatabaseConfigPersistence#updateConnectorDefinitions} method. */ @SuppressWarnings("PMD.SignatureDeclareThrowsException") -class DatabaseConfigPersistenceUpdateConnectorDefinitionsTest extends BaseDatabaseConfigPersistenceTest { +class DatabaseConfigPersistenceUpdateConnectorDefinitionsTest extends BaseConfigDatabaseTest { + + protected static final StandardSourceDefinition SOURCE_GITHUB = new StandardSourceDefinition() + .withName("GitHub") + .withSourceDefinitionId(UUID.fromString("ef69ef6e-aa7f-4af1-a01d-ef775033524e")) + .withDockerRepository("airbyte/source-github") + .withDockerImageTag("0.2.3") + .withDocumentationUrl("https://docs.airbyte.io/integrations/sources/github") + .withIcon("github.svg") + .withSourceType(SourceType.API) + .withProtocolVersion("0.2.0") + .withTombstone(false); private static final JsonNode SOURCE_GITHUB_JSON = Jsons.jsonNode(SOURCE_GITHUB); private static final String DOCKER_IMAGE_TAG = "0.0.0"; private static final String DOCKER_IMAGE_TAG_2 = "0.1000.0"; private static final String DEFAULT_PROTOCOL_VERSION = "0.2.0"; - @BeforeAll - public static void setup() throws Exception { - dataSource = DatabaseConnectionHelper.createDataSource(container); - dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); - flyway = FlywayFactory.create(dataSource, DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.class.getName(), - ConfigsDatabaseMigrator.DB_IDENTIFIER, - ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); - database = new ConfigsDatabaseTestProvider(dslContext, flyway).create(false); - configPersistence = new DatabaseConfigPersistence(database); - final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, flyway); - final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); - MigrationDevHelper.runLastMigration(devDatabaseMigrator); - } - - @AfterAll - public static void tearDown() throws Exception { - dslContext.close(); - DataSourceFactory.close(dataSource); - } + private DatabaseConfigPersistence configPersistence; @BeforeEach public void resetDatabase() throws SQLException { truncateAllTables(); + + configPersistence = new DatabaseConfigPersistence(database); } @Test @@ -227,4 +218,19 @@ private void assertUpdateConnectorDefinition(final List> recordCount = database.query(ctx -> ctx.select(count(asterisk())).from(table).fetch()); + assertEquals(expectedCount, recordCount.get(0).value1()); + } + + protected void assertHasSource(final StandardSourceDefinition source) throws Exception { + assertEquals(source, configPersistence + .getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), + StandardSourceDefinition.class)); + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java index 52b6e4f86e63..299d16657d0b 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -134,7 +134,7 @@ public class MockData { private static final Instant NOW = Instant.parse("2021-12-15T20:30:40.00Z"); - private static final String CONNECTION_SPECIFICATION = "'{\"name\":\"John\", \"age\":30, \"car\":null}'"; + private static final String CONNECTION_SPECIFICATION = "{\"name\":\"John\", \"age\":30, \"car\":null}"; private static final UUID OPERATION_ID_4 = UUID.randomUUID(); private static final UUID WEBHOOK_CONFIG_ID = UUID.randomUUID(); private static final String WEBHOOK_OPERATION_EXECUTION_URL = "test-webhook-url"; @@ -340,21 +340,21 @@ public static List sourceConnections() { .withTombstone(false) .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) .withWorkspaceId(WORKSPACE_ID_1) - .withConfiguration(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withConfiguration(Jsons.deserialize(CONNECTION_SPECIFICATION)) .withSourceId(SOURCE_ID_1); final SourceConnection sourceConnection2 = new SourceConnection() .withName("source-2") .withTombstone(false) .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) .withWorkspaceId(WORKSPACE_ID_1) - .withConfiguration(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withConfiguration(Jsons.deserialize(CONNECTION_SPECIFICATION)) .withSourceId(SOURCE_ID_2); final SourceConnection sourceConnection3 = new SourceConnection() .withName("source-3") .withTombstone(false) .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) .withWorkspaceId(WORKSPACE_ID_2) - .withConfiguration(Jsons.jsonNode((""))) + .withConfiguration(Jsons.emptyObject()) .withSourceId(SOURCE_ID_3); return Arrays.asList(sourceConnection1, sourceConnection2, sourceConnection3); } @@ -365,21 +365,21 @@ public static List destinationConnections() { .withTombstone(false) .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) .withWorkspaceId(WORKSPACE_ID_1) - .withConfiguration(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withConfiguration(Jsons.deserialize(CONNECTION_SPECIFICATION)) .withDestinationId(DESTINATION_ID_1); final DestinationConnection destinationConnection2 = new DestinationConnection() .withName("destination-2") .withTombstone(false) .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) .withWorkspaceId(WORKSPACE_ID_1) - .withConfiguration(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withConfiguration(Jsons.deserialize(CONNECTION_SPECIFICATION)) .withDestinationId(DESTINATION_ID_2); final DestinationConnection destinationConnection3 = new DestinationConnection() .withName("destination-3") .withTombstone(true) .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) .withWorkspaceId(WORKSPACE_ID_2) - .withConfiguration(Jsons.jsonNode("")) + .withConfiguration(Jsons.emptyObject()) .withDestinationId(DESTINATION_ID_3); return Arrays.asList(destinationConnection1, destinationConnection2, destinationConnection3); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceE2ETest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceE2ETest.java index 1aecba632e0f..3e15ebe9bba3 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceE2ETest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceE2ETest.java @@ -25,13 +25,7 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.StandardSync.Status; import io.airbyte.config.StandardWorkspace; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.FlywayFactory; -import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; -import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.sql.SQLException; @@ -43,16 +37,15 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -class StandardSyncPersistenceE2ETest extends BaseDatabaseConfigPersistenceTest { +class StandardSyncPersistenceE2ETest extends BaseConfigDatabaseTest { record StandardSyncProtocolVersionFlag(UUID standardSyncId, boolean unsupportedProtocolVersion) {} private ConfigRepository configRepository; + private StandardSyncPersistence standardSyncPersistence; UUID workspaceId; StandardWorkspace workspace; @@ -71,23 +64,12 @@ record StandardSyncProtocolVersionFlag(UUID standardSyncId, boolean unsupportedP @BeforeEach void beforeEach() throws Exception { - dataSource = DatabaseConnectionHelper.createDataSource(container); - dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); - flyway = FlywayFactory.create(dataSource, StandardSyncPersistenceE2ETest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, - ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); - database = new ConfigsDatabaseTestProvider(dslContext, flyway).create(true); truncateAllTables(); standardSyncPersistence = new StandardSyncPersistence(database); configRepository = new ConfigRepository(database); } - @AfterEach - void afterEach() throws Exception { - dslContext.close(); - DataSourceFactory.close(dataSource); - } - @Test void testClearUnsupportedProtocolVersionFlagFromSource() throws IOException, JsonValidationException, SQLException { createBaseObjects(); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java index 4e85864e61cd..78b5ce3a2489 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java @@ -16,34 +16,27 @@ import io.airbyte.config.State; import io.airbyte.config.StateType; import io.airbyte.config.StateWrapper; -import io.airbyte.db.ExceptionWrappingDatabase; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.FlywayFactory; import io.airbyte.db.init.DatabaseInitializationException; -import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; -import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; import io.airbyte.protocol.models.AirbyteGlobalState; import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.AirbyteStreamState; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.test.utils.DatabaseConnectionHelper; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; +import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.UUID; import org.jooq.JSONB; -import org.jooq.SQLDialect; import org.jooq.impl.DSL; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -class StatePersistenceTest extends BaseDatabaseConfigPersistenceTest { +class StatePersistenceTest extends BaseConfigDatabaseTest { private ConfigRepository configRepository; private StatePersistence statePersistence; @@ -55,6 +48,36 @@ class StatePersistenceTest extends BaseDatabaseConfigPersistenceTest { private static final String GLOBAL_STATE = "\"my global state\""; private static final String STATE = "state"; + @BeforeEach + void beforeEach() throws DatabaseInitializationException, IOException, JsonValidationException, SQLException { + truncateAllTables(); + + setupTestData(); + statePersistence = new StatePersistence(database); + } + + private void setupTestData() throws JsonValidationException, IOException { + configRepository = new ConfigRepository(database); + + final StandardWorkspace workspace = MockData.standardWorkspaces().get(0); + final StandardSourceDefinition sourceDefinition = MockData.publicSourceDefinition(); + final SourceConnection sourceConnection = MockData.sourceConnections().get(0); + final StandardDestinationDefinition destinationDefinition = MockData.publicDestinationDefinition(); + final DestinationConnection destinationConnection = MockData.destinationConnections().get(0); + final StandardSync sync = MockData.standardSyncs().get(0); + + configRepository.writeStandardWorkspaceNoSecrets(workspace); + configRepository.writeStandardSourceDefinition(sourceDefinition); + configRepository.writeSourceConnectionNoSecrets(sourceConnection); + configRepository.writeStandardDestinationDefinition(destinationDefinition); + configRepository.writeDestinationConnectionNoSecrets(destinationConnection); + configRepository.writeStandardSyncOperation(MockData.standardSyncOperations().get(0)); + configRepository.writeStandardSyncOperation(MockData.standardSyncOperations().get(1)); + configRepository.writeStandardSync(sync); + + connectionId = sync.getConnectionId(); + } + @Test void testReadingNonExistingState() throws IOException { Assertions.assertTrue(statePersistence.getCurrentState(UUID.randomUUID()).isEmpty()); @@ -444,7 +467,7 @@ void testStreamFullReset() throws IOException { } @Test - void testInconsistentTypeUpdates() throws IOException { + void testInconsistentTypeUpdates() throws IOException, SQLException { final StateWrapper streamState = new StateWrapper() .withStateType(StateType.STREAM) .withStateMessages(Arrays.asList( @@ -479,10 +502,13 @@ void testInconsistentTypeUpdates() throws IOException { // We should be guarded against those cases let's make sure we don't make things worse if we're in // an inconsistent state - dslContext.insertInto(DSL.table(STATE)) - .columns(DSL.field("id"), DSL.field("connection_id"), DSL.field("type"), DSL.field(STATE)) - .values(UUID.randomUUID(), connectionId, io.airbyte.db.instance.configs.jooq.generated.enums.StateType.GLOBAL, JSONB.valueOf("{}")) - .execute(); + database.transaction(ctx -> { + ctx.insertInto(DSL.table(STATE)) + .columns(DSL.field("id"), DSL.field("connection_id"), DSL.field("type"), DSL.field(STATE)) + .values(UUID.randomUUID(), connectionId, io.airbyte.db.instance.configs.jooq.generated.enums.StateType.GLOBAL, JSONB.valueOf("{}")) + .execute(); + return null; + }); Assertions.assertThrows(IllegalStateException.class, () -> statePersistence.updateOrCreateState(connectionId, streamState)); Assertions.assertThrows(IllegalStateException.class, () -> statePersistence.getCurrentState(connectionId)); } @@ -497,79 +523,22 @@ void testEnumsConversion() { } @Test - void testStatePersistenceLegacyReadConsistency() throws IOException { - final JsonNode jsonState = Jsons.deserialize("{\"my\": \"state\"}"); - final State state = new State().withState(jsonState); - configRepository.updateConnectionState(connectionId, state); - - final StateWrapper readStateWrapper = statePersistence.getCurrentState(connectionId).orElseThrow(); - Assertions.assertEquals(StateType.LEGACY, readStateWrapper.getStateType()); - Assertions.assertEquals(state.getState(), readStateWrapper.getLegacyState()); - } - - @Test - void testStatePersistenceLegacyWriteConsistency() throws IOException { + void testStatePersistenceLegacyWriteConsistency() throws IOException, SQLException { final JsonNode jsonState = Jsons.deserialize("{\"my\": \"state\"}"); final StateWrapper stateWrapper = new StateWrapper().withStateType(StateType.LEGACY).withLegacyState(jsonState); statePersistence.updateOrCreateState(connectionId, stateWrapper); // Making sure we still follow the legacy format - final List readStates = dslContext - .selectFrom(STATE) + final List readStates = database.transaction(ctx -> ctx.selectFrom(STATE) .where(DSL.field("connection_id").eq(connectionId)) .fetch().map(r -> Jsons.deserialize(r.get(DSL.field(STATE, JSONB.class)).data(), State.class)) - .stream().toList(); + .stream() + .toList()); Assertions.assertEquals(1, readStates.size()); Assertions.assertEquals(readStates.get(0).getState(), stateWrapper.getLegacyState()); } - @BeforeEach - void beforeEach() throws DatabaseInitializationException, IOException, JsonValidationException { - dataSource = DatabaseConnectionHelper.createDataSource(container); - dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); - flyway = FlywayFactory.create(dataSource, StatePersistenceTest.class.getName(), - ConfigsDatabaseMigrator.DB_IDENTIFIER, ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); - database = new ConfigsDatabaseTestProvider(dslContext, flyway).create(true); - setupTestData(); - - statePersistence = new StatePersistence(database); - } - - @AfterEach - void afterEach() { - // Making sure we reset between tests - dslContext.dropSchemaIfExists("public").cascade().execute(); - dslContext.createSchema("public").execute(); - dslContext.setSchema("public").execute(); - } - - private void setupTestData() throws JsonValidationException, IOException { - configRepository = new ConfigRepository( - new DatabaseConfigPersistence(database), - database, - new ActorDefinitionMigrator(new ExceptionWrappingDatabase(database)), - new StandardSyncPersistence(database)); - - final StandardWorkspace workspace = MockData.standardWorkspaces().get(0); - final StandardSourceDefinition sourceDefinition = MockData.publicSourceDefinition(); - final SourceConnection sourceConnection = MockData.sourceConnections().get(0); - final StandardDestinationDefinition destinationDefinition = MockData.publicDestinationDefinition(); - final DestinationConnection destinationConnection = MockData.destinationConnections().get(0); - final StandardSync sync = MockData.standardSyncs().get(0); - - configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeStandardSourceDefinition(sourceDefinition); - configRepository.writeSourceConnectionNoSecrets(sourceConnection); - configRepository.writeStandardDestinationDefinition(destinationDefinition); - configRepository.writeDestinationConnectionNoSecrets(destinationConnection); - configRepository.writeStandardSyncOperation(MockData.standardSyncOperations().get(0)); - configRepository.writeStandardSyncOperation(MockData.standardSyncOperations().get(1)); - configRepository.writeStandardSync(sync); - - connectionId = sync.getConnectionId(); - } - private StateWrapper clone(final StateWrapper state) { return switch (state.getStateType()) { case LEGACY -> new StateWrapper() diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StreamResetPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StreamResetPersistenceTest.java index f121051430a9..b4c266f6da36 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StreamResetPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StreamResetPersistenceTest.java @@ -8,49 +8,25 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.spy; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.FlywayFactory; -import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; -import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; -import io.airbyte.db.instance.development.DevDatabaseMigrator; -import io.airbyte.db.instance.development.MigrationDevHelper; import io.airbyte.protocol.models.StreamDescriptor; -import io.airbyte.test.utils.DatabaseConnectionHelper; import java.util.ArrayList; import java.util.List; import java.util.UUID; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class StreamResetPersistenceTest extends BaseDatabaseConfigPersistenceTest { +class StreamResetPersistenceTest extends BaseConfigDatabaseTest { static StreamResetPersistence streamResetPersistence; private static final Logger LOGGER = LoggerFactory.getLogger(StreamResetPersistenceTest.class); @BeforeEach public void setup() throws Exception { - dataSource = DatabaseConnectionHelper.createDataSource(container); - dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); - flyway = FlywayFactory.create(dataSource, StreamResetPersistenceTest.class.getName(), ConfigsDatabaseMigrator.DB_IDENTIFIER, - ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); - database = new ConfigsDatabaseTestProvider(dslContext, flyway).create(false); - streamResetPersistence = spy(new StreamResetPersistence(database)); - final ConfigsDatabaseMigrator configsDatabaseMigrator = - new ConfigsDatabaseMigrator(database, flyway); - final DevDatabaseMigrator devDatabaseMigrator = new DevDatabaseMigrator(configsDatabaseMigrator); - MigrationDevHelper.runLastMigration(devDatabaseMigrator); truncateAllTables(); - } - @AfterEach - void tearDown() throws Exception { - dslContext.close(); - DataSourceFactory.close(dataSource); + streamResetPersistence = spy(new StreamResetPersistence(database)); } @Test @@ -62,15 +38,15 @@ void testCreateSameResetTwiceOnlyCreateItOnce() throws Exception { streamResetPersistence.createStreamResets(connectionId, List.of(streamDescriptor1, streamDescriptor2)); final List result = streamResetPersistence.getStreamResets(connectionId); - LOGGER.info(String.valueOf(dslContext.selectFrom("stream_reset").fetch())); + LOGGER.info(database.query(ctx -> ctx.selectFrom("stream_reset").fetch().toString())); assertEquals(2, result.size()); streamResetPersistence.createStreamResets(connectionId, List.of(streamDescriptor1)); - LOGGER.info(String.valueOf(dslContext.selectFrom("stream_reset").fetch())); + LOGGER.info(database.query(ctx -> ctx.selectFrom("stream_reset").fetch().toString())); assertEquals(2, streamResetPersistence.getStreamResets(connectionId).size()); streamResetPersistence.createStreamResets(connectionId, List.of(streamDescriptor2)); - LOGGER.info(String.valueOf(dslContext.selectFrom("stream_reset").fetch())); + LOGGER.info(database.query(ctx -> ctx.selectFrom("stream_reset").fetch().toString())); assertEquals(2, streamResetPersistence.getStreamResets(connectionId).size()); } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index f968e607d95b..0ac2e4e4694c 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -39,7 +39,7 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 1.2.7 + dockerImageTag: 1.2.8 documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery icon: bigquery.svg normalizationRepository: airbyte/normalization @@ -55,7 +55,7 @@ - name: BigQuery (denormalized typed struct) destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-bigquery-denormalized - dockerImageTag: 1.2.7 + dockerImageTag: 1.2.8 documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery icon: bigquery.svg normalizationRepository: airbyte/normalization diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 60ded4ebaf5b..f7b240eb6b02 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -621,7 +621,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-bigquery:1.2.7" +- dockerImage: "airbyte/destination-bigquery:1.2.8" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery" connectionSpecification: @@ -831,7 +831,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.7" +- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.8" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery" connectionSpecification: diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 513cd1673eb7..d6f463c57861 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -109,7 +109,7 @@ - name: Asana sourceDefinitionId: d0243522-dccf-4978-8ba0-37ed47a0bdbf dockerRepository: airbyte/source-asana - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.com/integrations/sources/asana icon: asana.svg sourceType: api @@ -437,7 +437,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.72 + dockerImageTag: 0.2.74 documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api @@ -584,7 +584,7 @@ - name: Google Ads sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 dockerRepository: airbyte/source-google-ads - dockerImageTag: 0.2.4 + dockerImageTag: 0.2.5 documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads icon: google-adwords.svg sourceType: api @@ -983,7 +983,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 1.0.13 + dockerImageTag: 1.0.14 documentationUrl: https://docs.airbyte.com/integrations/sources/mysql icon: mysql.svg sourceType: database @@ -1221,7 +1221,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.25 + dockerImageTag: 1.0.28 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database @@ -1827,7 +1827,7 @@ - name: Zendesk Support sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 dockerRepository: airbyte/source-zendesk-support - dockerImageTag: 0.2.16 + dockerImageTag: 0.2.17 documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-support icon: zendesk.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 7f715feddb29..e1c2118971be 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1343,7 +1343,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-asana:0.1.4" +- dockerImage: "airbyte/source-asana:0.1.5" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -3396,7 +3396,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.72" +- dockerImage: "airbyte/source-facebook-marketing:0.2.74" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.com/integrations/sources/facebook-marketing" @@ -3651,7 +3651,10 @@ action_breakdowns: title: "Action Breakdowns" description: "A list of chosen action_breakdowns for action_breakdowns" - default: [] + default: + - "action_type" + - "action_target_id" + - "action_destination" type: "array" items: title: "ValidActionBreakdowns" @@ -4973,7 +4976,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-google-ads:0.2.4" +- dockerImage: "airbyte/source-google-ads:0.2.5" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads" connectionSpecification: @@ -8706,7 +8709,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:1.0.13" +- dockerImage: "airbyte/source-mysql:1.0.14" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql" connectionSpecification: @@ -11150,7 +11153,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.25" +- dockerImage: "airbyte/source-postgres:1.0.28" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -11387,7 +11390,7 @@ method: type: "string" const: "CDC" - order: 0 + order: 1 plugin: type: "string" title: "Plugin" @@ -11400,21 +11403,21 @@ - "pgoutput" - "wal2json" const: "pgoutput" - order: 1 + order: 2 replication_slot: type: "string" title: "Replication Slot" description: "A plugin logical replication slot. Read about replication slots." - order: 2 + order: 3 publication: type: "string" title: "Publication" description: "A Postgres publication used for consuming changes. Read\ \ about publications and replication identities." - order: 3 + order: 4 initial_waiting_seconds: type: "integer" title: "Initial Waiting Time in Seconds (Advanced)" @@ -11424,7 +11427,7 @@ \ initial waiting time." default: 300 - order: 4 + order: 5 min: 120 max: 1200 tunnel_method: @@ -15863,7 +15866,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-zendesk-support:0.2.16" +- dockerImage: "airbyte/source-zendesk-support:0.2.17" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/zendesk-support" connectionSpecification: diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index 4ea925d62747..775d7f5d0a46 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -10,5 +10,5 @@ RUN pip install --no-cache-dir . ENTRYPOINT ["uvicorn", "connector_builder.entrypoint:app", "--host", "0.0.0.0", "--port", "80"] -LABEL io.airbyte.version=0.40.21 +LABEL io.airbyte.version=0.40.22 LABEL io.airbyte.name=airbyte/connector-builder-server diff --git a/airbyte-connector-builder-server/build.gradle b/airbyte-connector-builder-server/build.gradle index f73ddb7ecadd..4f8bb1e08b0d 100644 --- a/airbyte-connector-builder-server/build.gradle +++ b/airbyte-connector-builder-server/build.gradle @@ -50,7 +50,7 @@ task prepareBuild(type: Copy) { from layout.projectDirectory.file(".") exclude '.*' exclude 'build' - + exclude '**/*.pyc' into layout.buildDirectory.dir("docker") } diff --git a/airbyte-connector-builder-server/setup.py b/airbyte-connector-builder-server/setup.py index 68919ca9c637..dfdeeb068a79 100644 --- a/airbyte-connector-builder-server/setup.py +++ b/airbyte-connector-builder-server/setup.py @@ -14,7 +14,7 @@ setup( name="connector-builder-server", - version="0.40.21", + version="0.40.22", description="", long_description=README, author="Airbyte", diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 5a051b9f3bbd..b2ceb58f5eea 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -10,7 +10,7 @@ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/s && chmod +x kubectl && mv kubectl /usr/local/bin/ # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.40.21 +ARG VERSION=0.40.22 ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-cron/Dockerfile b/airbyte-cron/Dockerfile index 09866bbe8cdf..dd69b3c25e9e 100644 --- a/airbyte-cron/Dockerfile +++ b/airbyte-cron/Dockerfile @@ -1,7 +1,7 @@ ARG JDK_IMAGE=airbyte/airbyte-base-java-image:1.0 FROM ${JDK_IMAGE} AS cron -ARG VERSION=0.40.21 +ARG VERSION=0.40.22 ENV APPLICATION airbyte-cron ENV VERSION ${VERSION} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTestProvider.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTestProvider.java index 98f09191e776..1c26d28c8287 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTestProvider.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/ConfigsDatabaseTestProvider.java @@ -41,8 +41,7 @@ public Database create(final boolean runMigration) throws IOException, DatabaseI final Database database = new Database(dslContext); if (runMigration) { - final DatabaseMigrator migrator = new ConfigsDatabaseMigrator( - database, flyway); + final DatabaseMigrator migrator = new ConfigsDatabaseMigrator(database, flyway); migrator.createBaseline(); migrator.migrate(); } else { diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java index f736d5c4a0b6..6e1ae4ec2e69 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java @@ -45,7 +45,11 @@ public static long getEstimatedByteSize(final Object rowData) { // the whole method only provides an estimation. Please never convert // the string to byte[] to get the exact length. That conversion is known // to introduce a lot of memory overhead. - return Jsons.serialize(rowData).length() * 4L; + // + // We are using 3L as the median byte-size of a serialized char here assuming that most chars fit + // into the ASCII space (fewer bytes) + + return Jsons.serialize(rowData).length() * 3L; } /** diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeConstants.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeConstants.java index 0689f700daba..d5a6d7615eaf 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeConstants.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeConstants.java @@ -10,7 +10,7 @@ public final class FetchSizeConstants { // This size is not enforced. It is only used to calculate a proper // fetch size. The max row size the connector can handle is actually // limited by the heap size. - public static final double TARGET_BUFFER_SIZE_RATIO = 0.5; + public static final double TARGET_BUFFER_SIZE_RATIO = 0.6; public static final long MIN_BUFFER_BYTE_SIZE = 250L * 1024L * 1024L; // 250 MB public static final long MAX_BUFFER_BYTE_SIZE = 1024L * 1024L * 1024L; // 1GB // sample size for making the first estimation of the row size diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimatorTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimatorTest.java index 63a931cfdf70..3046761bbbde 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimatorTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimatorTest.java @@ -16,8 +16,8 @@ class BaseSizeEstimatorTest { @Test void testGetEstimatedByteSize() { assertEquals(0L, BaseSizeEstimator.getEstimatedByteSize(null)); - assertEquals(28L, BaseSizeEstimator.getEstimatedByteSize("12345")); - assertEquals(60L, BaseSizeEstimator.getEstimatedByteSize(Jsons.jsonNode(Map.of("key", "value")))); + assertEquals(21L, BaseSizeEstimator.getEstimatedByteSize("12345")); + assertEquals(45L, BaseSizeEstimator.getEstimatedByteSize(Jsons.jsonNode(Map.of("key", "value")))); } public static class TestSizeEstimator extends BaseSizeEstimator { diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimatorTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimatorTest.java index d33045c5d5d9..1f61247af91a 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimatorTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimatorTest.java @@ -41,9 +41,9 @@ void testIt() { sizeEstimator.accept("11111"); final Optional fetchSize = sizeEstimator.getFetchSize(); assertTrue(fetchSize.isPresent()); - final long expectedMaxByteSize = 28L; + final long expectedMaxByteSize = 21L; assertEquals(expectedMaxByteSize, Math.round(sizeEstimator.getMaxRowByteSize())); - assertEquals(bufferByteSize / expectedMaxByteSize, fetchSize.get().longValue()); + assertEquals((bufferByteSize / expectedMaxByteSize) + 1, fetchSize.get().longValue()); // + 1 needed for int remainder rounding } } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimatorTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimatorTest.java index 2786bbe306f2..7cc57700ec99 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimatorTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimatorTest.java @@ -29,38 +29,38 @@ void testIt() { double maxByteSize = initialByteSize; - // size: 3 * 4 = 12, not sampled + // size: 3 * 3 = 12, not sampled sizeEstimator.accept("1"); assertFalse(sizeEstimator.getFetchSize().isPresent()); assertEquals(maxByteSize, sizeEstimator.getMaxRowByteSize()); - // size: 4 * 4 = 16, not sampled + // size: 4 * 3 = 16, not sampled sizeEstimator.accept("11"); assertFalse(sizeEstimator.getFetchSize().isPresent()); assertEquals(maxByteSize, sizeEstimator.getMaxRowByteSize()); - // size: 5 * 4 = 20, sampled, fetch size is ready + // size: 5 * 3 = 15, sampled, fetch size is ready sizeEstimator.accept("111"); final Optional fetchSize1 = sizeEstimator.getFetchSize(); - maxByteSize = 20; - assertDoubleEquals(20, sizeEstimator.getMaxRowByteSize()); + maxByteSize = 15; + assertDoubleEquals(15, sizeEstimator.getMaxRowByteSize()); assertDoubleEquals(bufferByteSize / maxByteSize, fetchSize1.get().doubleValue()); - // size: 6 * 4 = 24, not sampled + // size: 6 * 3 = 24, not sampled sizeEstimator.accept("1111"); assertFalse(sizeEstimator.getFetchSize().isPresent()); assertDoubleEquals(maxByteSize, sizeEstimator.getMaxRowByteSize()); - // size: 7 * 4 = 28, not sampled + // size: 7 * 3 = 28, not sampled sizeEstimator.accept("11111"); assertFalse(sizeEstimator.getFetchSize().isPresent()); assertDoubleEquals(maxByteSize, sizeEstimator.getMaxRowByteSize()); - // size: 8 * 4 = 32, sampled, fetch size is ready + // size: 8 * 3 = 24, sampled, fetch size is ready sizeEstimator.accept("111111"); final Optional fetchSize2 = sizeEstimator.getFetchSize(); assertTrue(fetchSize2.isPresent()); - maxByteSize = 32; + maxByteSize = 24; assertDoubleEquals(maxByteSize, sizeEstimator.getMaxRowByteSize()); assertDoubleEquals(bufferByteSize / maxByteSize, fetchSize2.get().doubleValue()); } diff --git a/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java index a010344bd394..8112f4b50927 100644 --- a/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java +++ b/airbyte-integrations/bases/debezium-v1-9-6/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java @@ -4,20 +4,38 @@ package io.airbyte.integrations.debezium.internals; +import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; +import static io.airbyte.db.jdbc.DateTimeConverter.convertToDate; +import static io.airbyte.db.jdbc.DateTimeConverter.convertToTime; +import static io.airbyte.db.jdbc.DateTimeConverter.convertToTimestamp; +import static io.airbyte.db.jdbc.DateTimeConverter.convertToTimestampWithTimezone; +import static org.apache.kafka.connect.data.Schema.OPTIONAL_BOOLEAN_SCHEMA; +import static org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA; +import static org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA; + import io.airbyte.db.jdbc.DateTimeConverter; import io.debezium.spi.converter.CustomConverter; import io.debezium.spi.converter.RelationalColumn; import io.debezium.time.Conversions; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; +import java.sql.SQLException; import java.time.LocalDate; import java.time.LocalTime; +import java.time.OffsetTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; +import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.Properties; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.commons.codec.binary.Hex; import org.apache.kafka.connect.data.SchemaBuilder; +import org.postgresql.jdbc.PgArray; import org.postgresql.util.PGInterval; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +51,7 @@ public class PostgresConverter implements CustomConverter s.equalsIgnoreCase(field.typeName()))) { registerNumber(field, registration); + } else if (Arrays.stream(ARRAY_TYPES).anyMatch(s -> s.equalsIgnoreCase(field.typeName()))) { + registerArray(field, registration); } } + private void registerArray(RelationalColumn field, ConverterRegistration registration) { + final String fieldType = field.typeName().toUpperCase(); + final SchemaBuilder arraySchema = switch (fieldType) { + case "_NUMERIC", "_MONEY" -> SchemaBuilder.array(OPTIONAL_FLOAT64_SCHEMA); + case "_NAME", "_DATE", "_TIME", "_TIMESTAMP", "_TIMESTAMPTZ", "_TIMETZ", "_BYTEA" -> SchemaBuilder.array(OPTIONAL_STRING_SCHEMA); + case "_BIT" -> SchemaBuilder.array(OPTIONAL_BOOLEAN_SCHEMA); + default -> SchemaBuilder.array(OPTIONAL_STRING_SCHEMA); + }; + registration.register(arraySchema, x -> convertArray(x, field)); + } + private void registerNumber(final RelationalColumn field, final ConverterRegistration registration) { registration.register(SchemaBuilder.string().optional(), x -> { if (x == null) { @@ -106,6 +138,72 @@ private void registerText(final RelationalColumn field, final ConverterRegistrat }); } + private Object convertArray(Object x, RelationalColumn field) { + final String fieldType = field.typeName().toUpperCase(); + Object[] values = new Object[0]; + try { + values = (Object[]) ((PgArray) x).getArray(); + } catch (SQLException e) { + LOGGER.error("Failed to convert PgArray:" + e); + } + switch (fieldType) { + // debezium currently cannot handle MONEY[] datatype and it's not implemented + case "_MONEY": + // PgArray.getArray() trying to convert to Double instead of PgMoney + // due to incorrect type mapping in the postgres driver + // https://github.com/pgjdbc/pgjdbc/blob/d5ed52ef391670e83ae5265af2f7301c615ce4ca/pgjdbc/src/main/java/org/postgresql/jdbc/TypeInfoCache.java#L88 + // and throws an exception, so a custom implementation of converting to String is used to get the + // value as is + final String nativeMoneyValue = ((PgArray) x).toString(); + final String substringM = Objects.requireNonNull(nativeMoneyValue).substring(1, nativeMoneyValue.length() - 1); + final char currency = substringM.charAt(0); + final String regex = "\\" + currency; + final List myListM = new ArrayList<>(Arrays.asList(substringM.split(regex))); + return myListM.stream() + // since the separator is the currency sign, all extra characters must be removed except for numbers + // and dots + .map(val -> val.replaceAll("[^\\d.]", "")) + .filter(money -> !money.isEmpty()) + .map(Double::valueOf) + .collect(Collectors.toList()); + case "_NUMERIC": + return Arrays.stream(values).map(value -> value == null ? null : Double.valueOf(value.toString())).collect(Collectors.toList()); + case "_TIME": + return Arrays.stream(values).map(value -> value == null ? null : convertToTime(value)).collect(Collectors.toList()); + case "_DATE": + return Arrays.stream(values).map(value -> value == null ? null : convertToDate(value)).collect(Collectors.toList()); + case "_TIMESTAMP": + return Arrays.stream(values).map(value -> value == null ? null : convertToTimestamp(value)).collect(Collectors.toList()); + case "_TIMESTAMPTZ": + return Arrays.stream(values).map(value -> value == null ? null : convertToTimestampWithTimezone(value)).collect(Collectors.toList()); + case "_TIMETZ": + + final List timetzArr = new ArrayList<>(); + final String nativeValue = ((PgArray) x).toString(); + final String substring = Objects.requireNonNull(nativeValue).substring(1, nativeValue.length() - 1); + final List times = new ArrayList<>(Arrays.asList(substring.split(","))); + final DateTimeFormatter format = DateTimeFormatter.ofPattern("HH:mm:ss[.SSSSSS]X"); + + times.forEach(s -> { + if (s.equalsIgnoreCase("NULL")) { + timetzArr.add(null); + } else { + final OffsetTime parsed = OffsetTime.parse(s, format); + timetzArr.add(parsed.format(TIMETZ_FORMATTER)); + } + }); + return timetzArr; + case "_BYTEA": + return Arrays.stream(values).map(value -> Base64.getEncoder().encodeToString((byte[]) value)).collect(Collectors.toList()); + case "_BIT": + return Arrays.stream(values).map(value -> (Boolean) value).collect(Collectors.toList()); + case "_NAME": + return Arrays.stream(values).map(value -> (String) value).collect(Collectors.toList()); + default: + return new ArrayList<>(); + } + } + private int getTimePrecision(final RelationalColumn field) { return field.scale().orElse(-1); } @@ -127,30 +225,20 @@ private void registerDate(final RelationalColumn field, final ConverterRegistrat case "TIMESTAMP": if (x instanceof final Long l) { if (getTimePrecision(field) <= 3) { - return DateTimeConverter.convertToTimestamp(Conversions.toInstantFromMillis(l)); + return convertToTimestamp(Conversions.toInstantFromMillis(l)); } if (getTimePrecision(field) <= 6) { - return DateTimeConverter.convertToTimestamp(Conversions.toInstantFromMicros(l)); + return convertToTimestamp(Conversions.toInstantFromMicros(l)); } } - return DateTimeConverter.convertToTimestamp(x); + return convertToTimestamp(x); case "DATE": if (x instanceof Integer) { - return DateTimeConverter.convertToDate(LocalDate.ofEpochDay((Integer) x)); + return convertToDate(LocalDate.ofEpochDay((Integer) x)); } - return DateTimeConverter.convertToDate(x); + return convertToDate(x); case "TIME": - if (x instanceof Long) { - if (getTimePrecision(field) <= 3) { - long l = Math.multiplyExact((Long) x, TimeUnit.MILLISECONDS.toNanos(1)); - return DateTimeConverter.convertToTime(LocalTime.ofNanoOfDay(l)); - } - if (getTimePrecision(field) <= 6) { - long l = Math.multiplyExact((Long) x, TimeUnit.MICROSECONDS.toNanos(1)); - return DateTimeConverter.convertToTime(LocalTime.ofNanoOfDay(l)); - } - } - return DateTimeConverter.convertToTime(x); + return resolveTime(field, x); case "INTERVAL": return convertInterval((PGInterval) x); default: @@ -159,6 +247,20 @@ private void registerDate(final RelationalColumn field, final ConverterRegistrat }); } + private String resolveTime(RelationalColumn field, Object x) { + if (x instanceof Long) { + if (getTimePrecision(field) <= 3) { + long l = Math.multiplyExact((Long) x, TimeUnit.MILLISECONDS.toNanos(1)); + return DateTimeConverter.convertToTime(LocalTime.ofNanoOfDay(l)); + } + if (getTimePrecision(field) <= 6) { + long l = Math.multiplyExact((Long) x, TimeUnit.MICROSECONDS.toNanos(1)); + return DateTimeConverter.convertToTime(LocalTime.ofNanoOfDay(l)); + } + } + return DateTimeConverter.convertToTime(x); + } + private String convertInterval(final PGInterval pgInterval) { final StringBuilder resultInterval = new StringBuilder(); formatDateUnit(resultInterval, pgInterval.getYears(), " year "); diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java b/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java index 96dd7b96db97..e16f04b1d2a1 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java @@ -10,7 +10,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.destination.s3.avro.JsonSchemaType; -import io.airbyte.integrations.standardtest.destination.NumberDataTypeTestArgumentProvider; +import io.airbyte.integrations.standardtest.destination.argproviders.NumberDataTypeTestArgumentProvider; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; diff --git a/airbyte-integrations/bases/source-acceptance-test/setup.py b/airbyte-integrations/bases/source-acceptance-test/setup.py index 9b9d7ffe8fa9..e5c4bb96e07f 100644 --- a/airbyte-integrations/bases/source-acceptance-test/setup.py +++ b/airbyte-integrations/bases/source-acceptance-test/setup.py @@ -13,7 +13,7 @@ "inflection~=0.5", "pdbpp~=0.10", "pydantic~=1.6", - "pytest~=6.1", + "pytest~=6.2", "pytest-sugar~=0.9", "pytest-timeout~=1.4", "pprintpp~=0.4", diff --git a/airbyte-integrations/bases/standard-destination-test/build.gradle b/airbyte-integrations/bases/standard-destination-test/build.gradle index 8086385c174e..d69a666b31fa 100644 --- a/airbyte-integrations/bases/standard-destination-test/build.gradle +++ b/airbyte-integrations/bases/standard-destination-test/build.gradle @@ -13,4 +13,5 @@ dependencies { implementation 'org.junit.jupiter:junit-jupiter-api' implementation 'org.junit.jupiter:junit-jupiter-params' implementation 'org.mockito:mockito-core:4.6.1' + } diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 899f7cb3ff00..a5e422a49030 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -4,6 +4,11 @@ package io.airbyte.integrations.standardtest.destination; +import static io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.INFINITY_TYPE_MESSAGE; +import static io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.INTEGER_TYPE_CATALOG; +import static io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.NAN_TYPE_MESSAGE; +import static io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.NUMBER_TYPE_CATALOG; +import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -27,6 +32,8 @@ import io.airbyte.config.StandardCheckConnectionOutput.Status; import io.airbyte.config.WorkerDestinationConfig; import io.airbyte.integrations.destination.NamingConventionTransformer; +import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; +import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.integrations.standardtest.destination.comparator.BasicTestDataComparator; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.AirbyteCatalog; @@ -71,6 +78,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.Builder; +import lombok.Getter; import org.joda.time.DateTime; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -429,13 +438,13 @@ public void testSecondSync() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( catalog); final List firstSyncMessages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() + DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final JsonNode config = getConfig(); @@ -446,7 +455,7 @@ public void testSecondSync() throws Exception { // So let's create a dummy data that will be checked after all sync. It should remain the same final AirbyteCatalog dummyCatalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); dummyCatalog.getStreams().get(0).setName(DUMMY_CATALOG_NAME); final ConfiguredAirbyteCatalog configuredDummyCatalog = CatalogHelpers.toDefaultConfiguredCatalog( @@ -495,7 +504,7 @@ public void testSecondSync() throws Exception { public void testLineBreakCharacters() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( catalog); @@ -560,7 +569,7 @@ public void testIncrementalSync() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( catalog); @@ -570,7 +579,7 @@ public void testIncrementalSync() throws Exception { }); final List firstSyncMessages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() + DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final JsonNode config = getConfig(); @@ -651,7 +660,7 @@ public void testIncrementalDedupeSync() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( catalog); @@ -665,7 +674,7 @@ public void testIncrementalDedupeSync() throws Exception { }); final List firstSyncMessages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() + DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final JsonNode config = getConfig(); @@ -754,12 +763,12 @@ void testSyncVeryBigRecords() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( catalog); final List messages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() + DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); // Add a big message that barely fits into the limits of the destination @@ -946,7 +955,7 @@ void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { // TODO(davin): make these tests part of the catalog file. final AirbyteCatalog catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); final String namespace = "sourcenamespace"; catalog.getStreams().forEach(stream -> stream.setNamespace(namespace)); @@ -954,7 +963,7 @@ void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { catalog); final List messages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() + DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final List messagesWithNewNamespace = getRecordMessagesWithNewNamespace( @@ -978,7 +987,7 @@ void testSyncWriteSameTableNameDifferentNamespace() throws Exception { // TODO(davin): make these tests part of the catalog file. final var catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); final var namespace1 = "sourcenamespace"; catalog.getStreams().forEach(stream -> stream.setNamespace(namespace1)); @@ -995,14 +1004,12 @@ void testSyncWriteSameTableNameDifferentNamespace() throws Exception { catalog.getStreams().addAll(diffNamespaceStreams); final var configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); - - final var ns1Messages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() + final var messageFile = DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getMessageFileVersion(getProtocolVersion()); + final var ns1Messages = MoreResources.readResource(messageFile).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final var ns1MessagesAtNamespace1 = getRecordMessagesWithNewNamespace(ns1Messages, namespace1); - final var ns2Messages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() + final var ns2Messages = MoreResources.readResource(messageFile).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final var ns2MessagesAtNamespace2 = getRecordMessagesWithNewNamespace(ns2Messages, namespace2); @@ -1016,23 +1023,6 @@ void testSyncWriteSameTableNameDifferentNamespace() throws Exception { retrieveRawRecordsAndAssertSameMessages(catalog, allMessages, defaultSchema); } - public static class NamespaceTestCaseProvider implements ArgumentsProvider { - - @Override - public Stream provideArguments(final ExtensionContext context) - throws Exception { - final JsonNode testCases = - Jsons.deserialize(MoreResources.readResource("namespace_test_cases.json")); - return MoreIterators.toList(testCases.elements()).stream() - .filter(testCase -> testCase.get("enabled").asBoolean()) - .map(testCase -> Arguments.of( - testCase.get("id").asText(), - testCase.get("namespace").asText(), - testCase.get("normalized").asText())); - } - - } - @ParameterizedTest @ArgumentsSource(NamespaceTestCaseProvider.class) public void testNamespaces(final String testCaseId, @@ -1049,14 +1039,14 @@ public void testNamespaces(final String testCaseId, } final AirbyteCatalog catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.NAMESPACE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.NAMESPACE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); catalog.getStreams().forEach(stream -> stream.setNamespace(namespace)); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( catalog); final List messages = MoreResources.readResource( - DataArgumentsProvider.NAMESPACE_CONFIG.messageFile).lines() + DataArgumentsProvider.NAMESPACE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final List messagesWithNewNamespace = getRecordMessagesWithNewNamespace( @@ -1105,13 +1095,13 @@ public void testSyncNotFailsWithNewFields() throws Exception { final AirbyteCatalog catalog = Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.catalogFile), + MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( catalog); final List firstSyncMessages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.messageFile).lines() + DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); final JsonNode config = getConfig(); @@ -1563,6 +1553,38 @@ protected boolean supportObjectDataTypeTest() { return false; } + /** + * NaN and Infinity test are not supported by default. Please override this method to specify + * NaN/Infinity types support example: + * + * 
+   *
+   * protected SpecialNumericTypes getSpecialNumericTypesSupportTest() {
+   *   return SpecialNumericTypes.builder()
+   *       .supportNumberNan(true)
+   *       .supportIntegerNan(true)
+   *       .build();
+   * }
+   *
+ * + * @return SpecialNumericTypes with support flags + */ + protected SpecialNumericTypes getSpecialNumericTypesSupportTest() { + return SpecialNumericTypes.builder().build(); + } + + /** + * The method should be overridden if destination connector support newer protocol version otherwise + * {@link io.airbyte.integrations.standardtest.destination.ProtocolVersion#V0} is used + *

+ * NOTE: Method should be public in a sake of java reflection + * + * @return + */ + public ProtocolVersion getProtocolVersion() { + return ProtocolVersion.V0; + } + private boolean checkTestCompatibility( final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) { return testCompatibility.isTestCompatible(supportBasicDataTypeTest(), @@ -1584,6 +1606,74 @@ public void testDataTypeTestWithNormalization(final String messagesFilename, catalog); final List messages = readMessagesFromFile(messagesFilename); + runAndCheck(catalog, configuredCatalog, messages); + } + + @Test + public void testSyncNumberNanDataType() throws Exception { + // NaN/Infinity protocol supports started from V1 version or higher + SpecialNumericTypes numericTypesSupport = getSpecialNumericTypesSupportTest(); + if (getProtocolVersion().equals(ProtocolVersion.V0) || !numericTypesSupport.isSupportNumberNan()) { + return; + } + final AirbyteCatalog catalog = readCatalogFromFile(prefixFileNameByVersion(NUMBER_TYPE_CATALOG, getProtocolVersion())); + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); + final List messages = readMessagesFromFile(prefixFileNameByVersion(NAN_TYPE_MESSAGE, getProtocolVersion())); + final JsonNode config = getConfig(); + final String defaultSchema = getDefaultSchema(config); + + runAndCheck(catalog, configuredCatalog, messages); + } + + @Test + public void testSyncIntegerNanDataType() throws Exception { + // NaN/Infinity protocol supports started from V1 version or higher + SpecialNumericTypes numericTypesSupport = getSpecialNumericTypesSupportTest(); + if (getProtocolVersion().equals(ProtocolVersion.V0) || !numericTypesSupport.isSupportIntegerNan()) { + return; + } + final AirbyteCatalog catalog = readCatalogFromFile(prefixFileNameByVersion(INTEGER_TYPE_CATALOG, getProtocolVersion())); + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); + final List messages = readMessagesFromFile(prefixFileNameByVersion(NAN_TYPE_MESSAGE, getProtocolVersion())); + final JsonNode config = getConfig(); + final String defaultSchema = getDefaultSchema(config); + + runAndCheck(catalog, configuredCatalog, messages); + } + + @Test + public void testSyncNumberInfinityDataType() throws Exception { + // NaN/Infinity protocol supports started from V1 version or higher + SpecialNumericTypes numericTypesSupport = getSpecialNumericTypesSupportTest(); + if (getProtocolVersion().equals(ProtocolVersion.V0) || !numericTypesSupport.isSupportNumberInfinity()) { + return; + } + final AirbyteCatalog catalog = readCatalogFromFile(prefixFileNameByVersion(NUMBER_TYPE_CATALOG, getProtocolVersion())); + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); + final List messages = readMessagesFromFile(prefixFileNameByVersion(INFINITY_TYPE_MESSAGE, getProtocolVersion())); + final JsonNode config = getConfig(); + final String defaultSchema = getDefaultSchema(config); + + runAndCheck(catalog, configuredCatalog, messages); + } + + @Test + public void testSyncIntegerInfinityDataType() throws Exception { + // NaN/Infinity protocol supports started from V1 version or higher + SpecialNumericTypes numericTypesSupport = getSpecialNumericTypesSupportTest(); + if (getProtocolVersion().equals(ProtocolVersion.V0) || !numericTypesSupport.isSupportIntegerInfinity()) { + return; + } + final AirbyteCatalog catalog = readCatalogFromFile(prefixFileNameByVersion(INTEGER_TYPE_CATALOG, getProtocolVersion())); + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); + final List messages = readMessagesFromFile(prefixFileNameByVersion(INFINITY_TYPE_MESSAGE, getProtocolVersion())); + final JsonNode config = getConfig(); + final String defaultSchema = getDefaultSchema(config); + + runAndCheck(catalog, configuredCatalog, messages); + } + + private void runAndCheck(AirbyteCatalog catalog, ConfiguredAirbyteCatalog configuredCatalog, List messages) throws Exception { if (supportsNormalization()) { LOGGER.info("Normalization is supported! Run test with normalization."); runAndCheckWithNormalization(messages, configuredCatalog, catalog); @@ -1639,4 +1729,39 @@ private static List getRecordMessagesWithNewNamespace( return airbyteMessages; } + /** + * Can be used in overridden {@link #getSpecialNumericTypesSupportTest() + * getSpecialNumericTypesSupportTest()} method to specify if connector supports Integer/Number NaN + * or Integer/Number Infinity types + */ + @Builder + @Getter + public static class SpecialNumericTypes { + + boolean supportIntegerNan = false; + boolean supportNumberNan = false; + boolean supportIntegerInfinity = false; + boolean supportNumberInfinity = false; + + } + + public static class NamespaceTestCaseProvider implements ArgumentsProvider { + + public static final String NAMESPACE_TEST_CASES_JSON = "namespace_test_cases.json"; + + @Override + public Stream provideArguments(final ExtensionContext context) + throws Exception { + final JsonNode testCases = + Jsons.deserialize(MoreResources.readResource(NAMESPACE_TEST_CASES_JSON)); + return MoreIterators.toList(testCases.elements()).stream() + .filter(testCase -> testCase.get("enabled").asBoolean()) + .map(testCase -> Arguments.of( + testCase.get("id").asText(), + testCase.get("namespace").asText(), + testCase.get("normalized").asText())); + } + + } + } diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/ProtocolVersion.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/ProtocolVersion.java new file mode 100644 index 000000000000..ba5318114ae7 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/ProtocolVersion.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.standardtest.destination; + +public enum ProtocolVersion { + + V0("v0"), + V1("v1"); + + private final String prefix; + + ProtocolVersion(String prefix) { + this.prefix = prefix; + } + + public String getPrefix() { + return prefix; + } + +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataArgumentsProvider.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java similarity index 58% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataArgumentsProvider.java rename to airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java index 1422ce8eb825..029d8d12d832 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataArgumentsProvider.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java @@ -2,8 +2,12 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.integrations.standardtest.destination.argproviders; +import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; +import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; + +import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import java.util.stream.Stream; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.provider.Arguments; @@ -24,10 +28,11 @@ public class DataArgumentsProvider implements ArgumentsProvider { new CatalogMessageTestConfigPair("namespace_catalog.json", "namespace_messages.txt"); @Override - public Stream provideArguments(final ExtensionContext context) { + public Stream provideArguments(final ExtensionContext context) throws Exception { + ProtocolVersion protocolVersion = getProtocolVersion(context); return Stream.of( - Arguments.of(EXCHANGE_RATE_CONFIG.messageFile, EXCHANGE_RATE_CONFIG.catalogFile), - Arguments.of(EDGE_CASE_CONFIG.messageFile, EDGE_CASE_CONFIG.catalogFile) + Arguments.of(EXCHANGE_RATE_CONFIG.getMessageFileVersion(protocolVersion), EXCHANGE_RATE_CONFIG.getCatalogFileVersion(protocolVersion)), + Arguments.of(EDGE_CASE_CONFIG.getMessageFileVersion(protocolVersion), EDGE_CASE_CONFIG.getCatalogFileVersion(protocolVersion)) // todo - need to use the new protocol to capture this. // Arguments.of("stripe_messages.txt", "stripe_schema.json") ); @@ -44,6 +49,14 @@ public CatalogMessageTestConfigPair(final String catalogFile, final String messa this.messageFile = messageFile; } + public String getCatalogFileVersion(ProtocolVersion protocolVersion) { + return prefixFileNameByVersion(catalogFile, protocolVersion); + } + + public String getMessageFileVersion(ProtocolVersion protocolVersion) { + return prefixFileNameByVersion(messageFile, protocolVersion); + } + } } diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java similarity index 82% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java rename to airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java index aec61c87fe31..2496d010725c 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DataTypeTestArgumentProvider.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java @@ -2,8 +2,11 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.integrations.standardtest.destination.argproviders; +import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; + +import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import java.util.stream.Stream; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.provider.Arguments; @@ -15,6 +18,10 @@ public class DataTypeTestArgumentProvider implements ArgumentsProvider { private static final Logger LOGGER = LoggerFactory.getLogger(DataTypeTestArgumentProvider.class); + public static final String INTEGER_TYPE_CATALOG = "data_type_integer_type_test_catalog.json"; + public static final String NUMBER_TYPE_CATALOG = "data_type_number_type_test_catalog.json"; + public static final String NAN_TYPE_MESSAGE = "nan_type_test_message.txt"; + public static final String INFINITY_TYPE_MESSAGE = "nan_type_test_message.txt"; public static final CatalogMessageTestConfigWithCompatibility BASIC_TEST = new CatalogMessageTestConfigWithCompatibility("data_type_basic_test_catalog.json", "data_type_basic_test_messages.txt", new TestCompatibility(true, false, false)); @@ -27,9 +34,11 @@ public class DataTypeTestArgumentProvider implements ArgumentsProvider { public static final CatalogMessageTestConfigWithCompatibility OBJECT_WITH_ARRAY_TEST = new CatalogMessageTestConfigWithCompatibility("data_type_array_object_test_catalog.json", "data_type_array_object_test_messages.txt", new TestCompatibility(true, true, true)); + private ProtocolVersion protocolVersion; @Override public Stream provideArguments(ExtensionContext context) throws Exception { + protocolVersion = getProtocolVersion(context); return Stream.of( getArguments(BASIC_TEST), getArguments(ARRAY_TEST), @@ -38,7 +47,8 @@ public Stream provideArguments(ExtensionContext context) th } private Arguments getArguments(CatalogMessageTestConfigWithCompatibility testConfig) { - return Arguments.of(testConfig.messageFile, testConfig.catalogFile, testConfig.testCompatibility); + return Arguments.of(testConfig.getMessageFileVersion(protocolVersion), testConfig.getCatalogFileVersion(protocolVersion), + testConfig.testCompatibility); } public record TestCompatibility(boolean requireBasicCompatibility, diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/NumberDataTypeTestArgumentProvider.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java similarity index 51% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/NumberDataTypeTestArgumentProvider.java rename to airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java index ce5239460bf8..b8d056c61d52 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/NumberDataTypeTestArgumentProvider.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java @@ -2,8 +2,12 @@ * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.integrations.standardtest.destination.argproviders; +import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; +import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; + +import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import java.util.stream.Stream; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.provider.Arguments; @@ -15,12 +19,18 @@ public class NumberDataTypeTestArgumentProvider implements ArgumentsProvider { public static final String NUMBER_DATA_TYPE_TEST_MESSAGES = "number_data_type_test_messages.txt"; public static final String NUMBER_DATA_TYPE_ARRAY_TEST_CATALOG = "number_data_type_array_test_catalog.json"; public static final String NUMBER_DATA_TYPE_ARRAY_TEST_MESSAGES = "number_data_type_array_test_messages.txt"; + private ProtocolVersion protocolVersion; @Override - public Stream provideArguments(ExtensionContext context) { + public Stream provideArguments(ExtensionContext context) throws Exception { + protocolVersion = getProtocolVersion(context); return Stream.of( - Arguments.of(NUMBER_DATA_TYPE_TEST_CATALOG, NUMBER_DATA_TYPE_TEST_MESSAGES), - Arguments.of(NUMBER_DATA_TYPE_ARRAY_TEST_CATALOG, NUMBER_DATA_TYPE_ARRAY_TEST_MESSAGES)); + getArguments(NUMBER_DATA_TYPE_TEST_CATALOG, NUMBER_DATA_TYPE_TEST_MESSAGES), + getArguments(NUMBER_DATA_TYPE_ARRAY_TEST_CATALOG, NUMBER_DATA_TYPE_ARRAY_TEST_MESSAGES)); + } + + private Arguments getArguments(final String catalogFile, final String messageFile) { + return Arguments.of(prefixFileNameByVersion(catalogFile, protocolVersion), prefixFileNameByVersion(messageFile, protocolVersion)); } } diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java new file mode 100644 index 000000000000..f70332c6336b --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.standardtest.destination.argproviders.util; + +import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import java.lang.reflect.Method; +import org.junit.jupiter.api.extension.ExtensionContext; + +public class ArgumentProviderUtil { + + private static final String PROTOCOL_VERSION_METHOD_NAME = "getProtocolVersion"; + + /** + * This method use + * {@link io.airbyte.integrations.standardtest.destination.ProtocolVersion#getPrefix()} to prefix + * the file name. + *

+ * example: + *

+ * filename.json -> v0/filename.json + * + * @param fileName the original file name + * @param protocolVersion supported protocol version + * @return filename with protocol version prefix + */ + public static String prefixFileNameByVersion(final String fileName, ProtocolVersion protocolVersion) { + return String.format("%s/%s", protocolVersion.getPrefix(), fileName); + } + + /** + * This method use reflection to get protocol version method from provided test context. + *

+ * NOTE: getProtocolVersion method should be public. + * + * @param context the context in which the current test is being executed. + * @return supported protocol version + */ + public static ProtocolVersion getProtocolVersion(ExtensionContext context) throws Exception { + Class c = context.getRequiredTestClass(); + // NOTE: Method should be public + Method m = c.getMethod(PROTOCOL_VERSION_METHOD_NAME); + return (ProtocolVersion) m.invoke(c.getDeclaredConstructor().newInstance()); + } + +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_object_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_object_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_object_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_object_test_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_object_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_array_test_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_basic_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_basic_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_basic_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_basic_test_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_basic_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_object_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_object_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_object_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/data_type_object_test_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_object_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/edge_case_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/edge_case_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/edge_case_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/edge_case_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/edge_case_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/exchange_rate_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/exchange_rate_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/exchange_rate_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/exchange_rate_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/exchange_rate_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/exchange_rate_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/exchange_rate_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/exchange_rate_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/namespace_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/namespace_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/namespace_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/namespace_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_array_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_array_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_array_test_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_array_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_array_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_array_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_array_test_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_array_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_test_catalog.json rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/number_data_type_test_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/stripe_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/stripe_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/stripe_messages.txt rename to airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/stripe_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_catalog.json new file mode 100644 index 000000000000..8cf46426a146 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_catalog.json @@ -0,0 +1,47 @@ +{ + "streams": [ + { + "name": "object_array_test_1", + "json_schema": { + "type": ["object"], + "properties": { + "property_string": { + "$ref": "WellKnownTypes.json#/definitions/String" + }, + "property_array": { + "type": ["array"], + "items": { + "type": ["object"], + "properties": { + "property_string": { + "$ref": "WellKnownTypes.json#/definitions/String" + }, + "property_date": { + "$ref": "WellKnownTypes.json#/definitions/Date" + }, + "property_timestamp_with_timezone": { + "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + }, + "property_timestamp_without_timezone": { + "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" + }, + "property_number": { + "$ref": "WellKnownTypes.json#/definitions/Number" + }, + "property_integer": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + }, + "property_boolean": { + "$ref": "WellKnownTypes.json#/definitions/Boolean" + }, + "property_binary_data": { + "$ref": "WellKnownTypes.json#/definitions/BinaryData" + } + } + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_messages.txt new file mode 100644 index 000000000000..1f217a5dc1bc --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_messages.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "object_array_test_1", "emitted_at": 1602637589100, "data": { "property_string" : "qqq", "property_array" : [ { "property_string": "foo bar", "property_date": "2021-01-23", "property_timestamp_with_timezone": "2022-11-22T01:23:45+00:00", "property_timestamp_without_timezone": "2022-11-22T01:23:45", "property_number": "56.78", "property_integer": "42", "property_boolean": true, "property_binary_data" : "dGVzdA==" } ] }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_catalog.json new file mode 100644 index 000000000000..6b2d0d2d19cb --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_catalog.json @@ -0,0 +1,67 @@ +{ + "streams": [ + { + "name": "array_test_1", + "json_schema": { + "properties": { + "string_array": { + "type": ["array"], + "items": [ + { + "$ref": "WellKnownTypes.json#/definitions/String" + } + ] + }, + "array_date": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/Date" + } + }, + "array_timestamp_with_timezone": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + } + }, + "array_timestamp_without_timezone": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" + } + }, + "array_number": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + }, + "array_big_number": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + }, + "array_integer": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + } + }, + "array_boolean": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/Boolean" + } + }, + "array_binary_data": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/BinaryData" + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_messages.txt new file mode 100644 index 000000000000..5cc391b59b99 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_messages.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "array_test_1", "emitted_at": 1602637589100, "data": { "string_array" : ["foo bar", "some random special characters: ࠈൡሗ"], "array_date" : ["2021-01-23", "1504-02-29"], "array_timestamp_with_timezone" : ["2022-11-22T01:23:45+05:00", "9999-12-21T01:23:45-05:00"], "array_timestamp_without_timezone" : ["2022-11-22T01:23:45", "1504-02-29T01:23:45"], "array_number" : ["56.78", "0", "-12345.678"], "array_big_number" : ["-12345.678", "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234"], "array_integer" : ["42", "0", "12345"], "array_boolean" : [true, false], "array_binary_data" : ["dGVzdA=="] }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_catalog.json new file mode 100644 index 000000000000..6ad31482ab73 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_catalog.json @@ -0,0 +1,114 @@ +{ + "streams": [ + { + "name": "string_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "date_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Date" + } + } + } + }, + { + "name": "datetime_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + } + } + } + }, + { + "name": "datetime_test_2", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" + } + } + } + }, + { + "name": "time_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone" + } + } + } + }, + { + "name": "time_test_2", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/TimeWithoutTimezone" + } + } + } + }, + { + "name": "number_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + } + } + }, + { + "name": "bignumber_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + } + } + }, + { + "name": "integer_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + } + } + } + }, + { + "name": "boolean_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Boolean" + } + } + } + }, + { + "name": "binary_test_1", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/BinaryData" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_messages.txt new file mode 100644 index 000000000000..e3cb1d861919 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_messages.txt @@ -0,0 +1,28 @@ +{"type": "RECORD", "record": {"stream": "string_test_1", "emitted_at": 1602637589100, "data": { "data" : "foo bar" }}} +{"type": "RECORD", "record": {"stream": "string_test_1", "emitted_at": 1602637589200, "data": { "data" : "" }}} +{"type": "RECORD", "record": {"stream": "string_test_1", "emitted_at": 1602637589300, "data": { "data" : "some random special characters: ࠈൡሗ" }}} +{"type": "RECORD", "record": {"stream": "date_test_1", "emitted_at": 1602637589100, "data": { "data" : "2021-01-23" }}} +{"type": "RECORD", "record": {"stream": "date_test_1", "emitted_at": 1602637589200, "data": { "data" : "1504-02-29" }}} +{"type": "RECORD", "record": {"stream": "date_test_1", "emitted_at": 1602637589300, "data": { "data" : "9999-12-23" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_1", "emitted_at": 1602637589100, "data": { "data" : "2022-01-23T01:23:45Z" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_1", "emitted_at": 1602637589200, "data": { "data" : 2022-01-23T01:23:45.678-11:30 BC }}} +{"type": "RECORD", "record": {"stream": "datetime_test_1", "emitted_at": 1602637589300, "data": { "data" : "9999-12-23T01:23:45Z" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_2", "emitted_at": 1602637589100, "data": { "data" : "2022-11-22T01:23:45" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_2", "emitted_at": 1602637589200, "data": { "data" : "1504-02-29T01:23:45" }}} +{"type": "RECORD", "record": {"stream": "datetime_test_2", "emitted_at": 1602637589300, "data": { "data" : "9999-12-21T01:23:45" }}} +{"type": "RECORD", "record": {"stream": "time_test_1", "emitted_at": 1602637589100, "data": { "data" : "01:23:45Z" }}} +{"type": "RECORD", "record": {"stream": "time_test_1", "emitted_at": 1602637589200, "data": { "data" : "01:23:45.678-11:30" }}} +{"type": "RECORD", "record": {"stream": "time_test_2", "emitted_at": 1602637589300, "data": { "data" : "01:23:45" }}} +{"type": "RECORD", "record": {"stream": "time_test_2", "emitted_at": 1602637589100, "data": { "data" : "01:23:45.678" }}} +{"type": "RECORD", "record": {"stream": "number_test_1", "emitted_at": 1602637589200, "data": { "data" : "56.78" }}} +{"type": "RECORD", "record": {"stream": "number_test_1", "emitted_at": 1602637589300, "data": { "data" : "0" }}} +{"type": "RECORD", "record": {"stream": "number_test_1", "emitted_at": 1602637589100, "data": { "data" : "-12345.678" }}} +{"type": "RECORD", "record": {"stream": "bignumber_test_1", "emitted_at": 1602637589200, "data": { "data" : "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234" }}} +{"type": "RECORD", "record": {"stream": "bignumber_test_1", "emitted_at": 1602637589300, "data": { "data" : "0" }}} +{"type": "RECORD", "record": {"stream": "bignumber_test_1", "emitted_at": 1602637589100, "data": { "data" : "-12345.678" }}} +{"type": "RECORD", "record": {"stream": "integer_test_1", "emitted_at": 1602637589200, "data": { "data" : "42" }}} +{"type": "RECORD", "record": {"stream": "integer_test_1", "emitted_at": 1602637589300, "data": { "data" : "0" }}} +{"type": "RECORD", "record": {"stream": "integer_test_1", "emitted_at": 1602637589100, "data": { "data" : "-12345" }}} +{"type": "RECORD", "record": {"stream": "boolean_test_1", "emitted_at": 1602637589200, "data": { "data" : true }}} +{"type": "RECORD", "record": {"stream": "binary_test_1", "emitted_at": 1602637589300, "data": { "data" : "dGVzdA==" }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_integer_type_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_integer_type_test_catalog.json new file mode 100644 index 000000000000..8add7661b7c5 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_integer_type_test_catalog.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "name": "nan_test", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_number_type_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_number_type_test_catalog.json new file mode 100644 index 000000000000..2dee4beb6957 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_number_type_test_catalog.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "name": "nan_test", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_catalog.json new file mode 100644 index 000000000000..3ca962affe2e --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "object_test_1", + "json_schema": { + "type": ["object"], + "properties": { + "property_string": { + "$ref": "WellKnownTypes.json#/definitions/String" + }, + "property_date": { + "$ref": "WellKnownTypes.json#/definitions/Date" + }, + "property_timestamp_with_timezone": { + "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + }, + "property_timestamp_without_timezone": { + "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" + }, + "property_number": { + "$ref": "WellKnownTypes.json#/definitions/Number" + }, + "property_big_number": { + "$ref": "WellKnownTypes.json#/definitions/String" + }, + "property_integer": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + }, + "property_boolean": { + "$ref": "WellKnownTypes.json#/definitions/Boolean" + }, + "property_binary_data": { + "$ref": "WellKnownTypes.json#/definitions/BinaryData" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_messages.txt new file mode 100644 index 000000000000..c264eed78c94 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_messages.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "object_test_1", "emitted_at": 1602637589100, "data": {"property_string": "foo bar", "property_date": "2021-01-23", "property_timestamp_with_timezone": "2022-11-22T01:23:45+00:00", "property_timestamp_without_timezone": "2022-11-22T01:23:45", "property_number": "56.78", "property_big_number": "100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.1234", "property_integer": "42", "property_boolean": true, "property_binary_data" : "dGVzdA==" }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_catalog.json new file mode 100644 index 000000000000..e51a90c2386a --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_catalog.json @@ -0,0 +1,143 @@ +{ + "streams": [ + { + "name": "streamWithCamelCase", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + + { + "name": "stream_with_underscores", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "STREAM_WITH_ALL_CAPS", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "stream_with_edge_case_field_names", + "json_schema": { + "properties": { + "fieldWithCamelCase": { + "$ref": "WellKnownTypes.json#/definitions/String" + }, + "field_with_underscore": { + "$ref": "WellKnownTypes.json#/definitions/String" + }, + "FIELD_WITH_ALL_CAPS": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "stream-with:spécial:character_names", + "json_schema": { + "properties": { + "field_with_spécial_character": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "CapitalCase", + "json_schema": { + "properties": { + "deleted": { + "$ref": "WellKnownTypes.json#/definitions/Boolean" + } + } + } + }, + { + "name": "reserved_keywords", + "json_schema": { + "properties": { + "order": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "groups", + "json_schema": { + "properties": { + "authorization": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "ProperCase", + "json_schema": { + "properties": { + "ProperCase": { + "$ref": "WellKnownTypes.json#/definitions/Boolean" + } + } + } + }, + { + "name": "stream_name", + "json_schema": { + "properties": { + "some_id": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + }, + "some_field": { + "$ref": "WellKnownTypes.json#/definitions/String" + }, + "some_next_field": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "stream_name_next", + "json_schema": { + "properties": { + "some_id": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + }, + "next_field_name": { + "$ref": "WellKnownTypes.json#/definitions/String" + } + } + } + }, + { + "name": "stream_with_binary_data", + "json_schema": { + "properties": { + "some_id": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + }, + "binary_field_name": { + "$ref": "WellKnownTypes.json#/definitions/BinaryData" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_messages.txt new file mode 100644 index 000000000000..80039c75c62b --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_messages.txt @@ -0,0 +1,18 @@ +{"type": "RECORD", "record": {"stream": "streamWithCamelCase", "emitted_at": 1602637589000, "data": { "data" : "one" }}} +{"type": "RECORD", "record": {"stream": "stream_with_underscores", "emitted_at": 1602637589100, "data": { "data" : "one" }}} +{"type": "RECORD", "record": {"stream": "stream_with_edge_case_field_names", "emitted_at": 1602637589200, "data": { "fieldWithCamelCase" : "one" }}} +{"type": "RECORD", "record": {"stream": "stream_with_edge_case_field_names", "emitted_at": 1602637589300, "data": { "field_with_underscore" : "one" }}} +{"type": "RECORD", "record": {"stream": "stream-with:spécial:character_names", "emitted_at": 1602637589400, "data": { "field_with_spécial_character" : "one" }}} +{"type": "RECORD", "record": {"stream": "CapitalCase", "emitted_at": 1602637589000, "data": { "deleted" : true }}} +{"type": "RECORD", "record": {"stream": "reserved_keywords", "emitted_at": 1602637589000, "data": { "order" : "ascending" }}} +{"type": "RECORD", "record": {"stream": "groups", "emitted_at": 1602637589000, "data": { "authorization" : "into" }}} +{"type": "RECORD", "record": {"stream": "ProperCase", "emitted_at": 1602637589000, "data": { "ProperCase" : true }}} +{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589200, "data": { "some_id" : "101", "some_field" : "some_field_1", "some_next_field" : "some_next_field_1" }}} +{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589250, "data": { "some_id" : "102", "some_field" : "some_field_2" }}} +{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589300, "data": { "some_id" : "103", "some_next_field" : "some_next_field_3" }}} +{"type": "RECORD", "record": {"stream": "stream_name", "emitted_at": 1602637589350, "data": { "some_id" : "104" }}} +{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589400, "data": { "some_id" : "201", "next_field_name" : "next_field_name_1" }}} +{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589450, "data": { "some_id" : "202", "next_field_name" : "next_field_name_2" }}} +{"type": "RECORD", "record": {"stream": "stream_name_next", "emitted_at": 1602637589500, "data": { "some_id" : "203" }}} +{"type": "RECORD", "record": {"stream": "stream_with_binary_data", "emitted_at": 1602637589500, "data": { "some_id" : "303", "binary_field_name":"dGVzdA==" }}} +{"type": "STATE", "state": { "data": {"start_date": "2020-09-02"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_catalog.json new file mode 100644 index 000000000000..bfb3b03e14fc --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_catalog.json @@ -0,0 +1,29 @@ +{ + "streams": [ + { + "name": "exchange_rate", + "json_schema": { + "properties": { + "id": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + }, + "currency": { + "$ref": "WellKnownTypes.json#/definitions/String" + }, + "date": { + "$ref": "WellKnownTypes.json#/definitions/Date" + }, + "HKD": { + "$ref": "WellKnownTypes.json#/definitions/Number" + }, + "NZD": { + "$ref": "WellKnownTypes.json#/definitions/Number" + }, + "USD": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_messages.txt new file mode 100644 index 000000000000..7eddc0d31ad3 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_messages.txt @@ -0,0 +1,8 @@ +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637589000, "data": { "id": "1", "currency": "USD", "date": "2020-08-29T00:00:00Z", "NZD": "0.12", "HKD": "2.13"}}} +{"type": "STATE", "state": { "data": {"start_date": "2020-08-31"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637689100, "data": { "id": "1", "currency": "USD", "date": "2020-08-30T00:00:00Z", "NZD": "1.14", "HKD": "7.15"}}} +{"type": "STATE", "state": { "data": {"start_date": "2020-09-01"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637789200, "data": { "id": "2", "currency": "EUR", "date": "2020-08-31T00:00:00Z", "NZD": "1.14", "HKD": "7.15", "USD": "10.16"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637889300, "data": { "id": "2", "currency": "EUR", "date": "2020-08-31T00:00:00Z", "NZD": "1.14", "HKD": "7.99", "USD": "10.99"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637989400, "data": { "id": "2", "currency": "EUR", "date": "2020-09-01T00:00:00Z", "NZD": "1.14", "HKD": "7.15", "USD": "10.16"}}} +{"type": "STATE", "state": { "data": {"start_date": "2020-09-02"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/infinity_type_test_message.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/infinity_type_test_message.txt new file mode 100644 index 000000000000..34e17b29bb81 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/infinity_type_test_message.txt @@ -0,0 +1,3 @@ +{"type": "RECORD", "record": {"stream": "infinity", "emitted_at": 1602637589100, "data": { "data" : "Infinity" }}} +{"type": "RECORD", "record": {"stream": "infinity", "emitted_at": 1602637589200, "data": { "data" : "-Infinity" }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_catalog.json new file mode 100644 index 000000000000..9b6a53ff33f4 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_catalog.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "name": "data_stream", + "json_schema": { + "properties": { + "field1": { + "$ref": "WellKnownTypes.json#/definitions/Boolean" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_messages.txt new file mode 100644 index 000000000000..e40a257741e4 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_messages.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "data_stream", "emitted_at": 1602637589000, "data": { "field1" : true }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-08-17"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/nan_type_test_message.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/nan_type_test_message.txt new file mode 100644 index 000000000000..83cfc6b45a21 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/nan_type_test_message.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "nan_test", "emitted_at": 1602637589100, "data": { "data" : "NaN" }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_catalog.json new file mode 100644 index 000000000000..1cf69a9dd39a --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_catalog.json @@ -0,0 +1,29 @@ +{ + "streams": [ + { + "name": "array_test_1", + "json_schema": { + "properties": { + "array_number": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + }, + "array_float": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + }, + "array_integer": { + "type": ["array"], + "items": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_messages.txt new file mode 100644 index 000000000000..3b981b9d2a86 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_messages.txt @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "array_test_1", "emitted_at": 1602637589100, "data": { "array_number" : ["-12345.678", "100000000000000000.1234"],"array_float" : ["-12345.678", "0", "1000000000000000000000000000000000000000000000000000.1234"], "array_integer" : ["42", "0", "12345"]}}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_catalog.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_catalog.json new file mode 100644 index 000000000000..3892a1436cfd --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_catalog.json @@ -0,0 +1,34 @@ +{ + "streams": [ + { + "name": "int_test", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Integer" + } + } + } + }, + { + "name": "float_test", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + } + } + }, + { + "name": "default_number_test", + "json_schema": { + "properties": { + "data": { + "$ref": "WellKnownTypes.json#/definitions/Number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_messages.txt new file mode 100644 index 000000000000..1b17eb58284a --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_messages.txt @@ -0,0 +1,10 @@ +{"type": "RECORD", "record": {"stream": "int_test", "emitted_at": 1602637589100, "data": { "data" : "42" }}} +{"type": "RECORD", "record": {"stream": "int_test", "emitted_at": 1602637589200, "data": { "data" : "0" }}} +{"type": "RECORD", "record": {"stream": "int_test", "emitted_at": 1602637589300, "data": { "data" : "-12345" }}} +{"type": "RECORD", "record": {"stream": "float_test", "emitted_at": 1602637589100, "data": { "data" : "56.78" }}} +{"type": "RECORD", "record": {"stream": "float_test", "emitted_at": 1602637589200, "data": { "data" : "0" }}} +{"type": "RECORD", "record": {"stream": "float_test", "emitted_at": 1602637589300, "data": { "data" : "-12345.678" }}} +{"type": "RECORD", "record": {"stream": "default_number_test", "emitted_at": 1602637589100, "data": { "data" : "10000000000000000000000.1234" }}} +{"type": "RECORD", "record": {"stream": "default_number_test", "emitted_at": 1602637589200, "data": { "data" : "0" }}} +{"type": "RECORD", "record": {"stream": "default_number_test", "emitted_at": 1602637589300, "data": { "data" : "-12345.678" }}} +{"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/stripe_messages.txt b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/stripe_messages.txt new file mode 100644 index 000000000000..9f660bdb673f --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/stripe_messages.txt @@ -0,0 +1,3 @@ +{"type": "RECORD", "record": { "stream": "customers", "emitted_at": 1602637450, "data": {"id": "cus_I1l0XHrjLYwLR2", "object": "customer", "account_balance": "0", "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 3", "discount": null, "email": "customer3@test.com", "invoice_prefix": "EC156D8F", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"}} +{"type": "RECORD", "record": { "stream": "customers", "emitted_at": 1602637460, "data": {"id": "cus_I1l0INzfeSf2MM", "object": "customer", "account_balance": "0", "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 2", "discount": null, "email": "customer2@test.com", "invoice_prefix": "D4564D22", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"}} +{"type": "RECORD", "record": { "stream": "customers", "emitted_at": 1602637470, "data": {"id": "cus_I1l0cRVFy4ZhwQ", "object": "customer", "account_balance": "0", "created": "2020-09-15T16:58:52.000000Z", "currency": null, "default_source": null, "delinquent": false, "description": "Customer 1", "discount": null, "email": "customer1@test.com", "invoice_prefix": "92A8C396", "livemode": false, "metadata": {}, "shipping": null, "sources": [], "subscriptions": null, "tax_info": null, "tax_info_verification": null, "updated": "2020-09-15T16:58:52.000000Z"}, "time_extracted": "2020-09-15T18:01:23.634272Z"}} diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java index fcc0f7953482..6dd9c07abcca 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java @@ -117,7 +117,7 @@ public void testDataTypes() throws Exception { testDataHolders.forEach(testDataHolder -> { if (testCatalog()) { final AirbyteStream airbyteStream = streams.get(testDataHolder.getNameWithTestPrefix()); - final Map jsonSchemaTypeMap = (Map) Jsons.deserialize( + final Map jsonSchemaTypeMap = (Map) Jsons.deserialize( airbyteStream.getJsonSchema().get("properties").get(getTestColumnName()).toString(), Map.class); assertEquals(testDataHolder.getAirbyteType().getJsonSchemaTypeMap(), jsonSchemaTypeMap, "Expected column type for " + testDataHolder.getNameWithTestPrefix()); diff --git a/airbyte-integrations/connector-templates/destination-python/setup.py b/airbyte-integrations/connector-templates/destination-python/setup.py index fc0ed9135858..1a550e611aff 100644 --- a/airbyte-integrations/connector-templates/destination-python/setup.py +++ b/airbyte-integrations/connector-templates/destination-python/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = ["pytest~=6.1"] +TEST_REQUIREMENTS = ["pytest~=6.2"] setup( name="destination_{{snakeCase name}}", diff --git a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs index 2bba61e116a7..3012c8c027d9 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs @@ -10,7 +10,7 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", "source-acceptance-test", ] diff --git a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs index cc7cbf02ba6c..4c12bfa25fc3 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs @@ -10,7 +10,7 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", "source-acceptance-test", ] diff --git a/airbyte-integrations/connector-templates/source-python/setup.py.hbs b/airbyte-integrations/connector-templates/source-python/setup.py.hbs index 2c78aeb2ae2d..0476541f00a1 100644 --- a/airbyte-integrations/connector-templates/source-python/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/setup.py.hbs @@ -10,7 +10,7 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "source-acceptance-test", ] diff --git a/airbyte-integrations/connector-templates/source-singer/setup.py.hbs b/airbyte-integrations/connector-templates/source-singer/setup.py.hbs index d2049bccd395..10fa5ef2d6e1 100644 --- a/airbyte-integrations/connector-templates/source-singer/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-singer/setup.py.hbs @@ -11,7 +11,7 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 0ad3bb307bcf..a32f0bd1181b 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.7 +LABEL io.airbyte.version=1.2.8 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java index 66c30fcd3061..2c84a7806a33 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java @@ -35,8 +35,8 @@ import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DataArgumentsProvider; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index fccbfd8aae01..575b8fe4b92a 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.7 +LABEL io.airbyte.version=1.2.8 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index 2baaf6b4a970..e6c9c68c52f5 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -11,12 +11,15 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.api.gax.rpc.HeaderProvider; import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.Clustering; import com.google.cloud.bigquery.Dataset; import com.google.cloud.bigquery.DatasetInfo; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobId; import com.google.cloud.bigquery.JobInfo; @@ -25,12 +28,14 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TimePartitioning; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.config.WorkerEnvConstants; import io.airbyte.integrations.base.JavaBaseConstants; @@ -45,6 +50,7 @@ import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -64,7 +70,8 @@ public class BigQueryUtils { DateTimeFormatter.ofPattern("[yyyy][yy]['-']['/']['.'][' '][MMM][MM][M]['-']['/']['.'][' '][dd][d]" + "[[' ']['T']HH:mm[':'ss[.][SSSSSS][SSSSS][SSSS][SSS][' '][z][zzz][Z][O][x][XXX][XX][X]]]"); private static final String USER_AGENT_FORMAT = "%s (GPN: Airbyte)"; - private static final String CHECK_TEST_DATASET_SUFFIX = "_airbyte_check_stage_tmp"; + private static final String CHECK_TEST_DATASET_SUFFIX = "_airbyte_check_stage_tmp_" + System.currentTimeMillis(); + private static final String CHECK_TEST_TMP_TABLE_NAME = "test_connection_table_name"; public static ImmutablePair executeQuery(final BigQuery bigquery, final QueryJobConfiguration queryConfig) { final JobId jobId = JobId.of(UUID.randomUUID().toString()); @@ -119,16 +126,67 @@ public static Dataset getOrCreateDataset(final BigQuery bigquery, final String d public static void checkHasCreateAndDeleteDatasetRole(final BigQuery bigquery, final String datasetId, final String datasetLocation) { final String tmpTestDatasetId = datasetId + CHECK_TEST_DATASET_SUFFIX; - final Dataset dataset = bigquery.getDataset(tmpTestDatasetId); + final DatasetInfo datasetInfo = DatasetInfo.newBuilder(tmpTestDatasetId).setLocation(datasetLocation).build(); + + bigquery.create(datasetInfo); - // remove possible tmp datasets from previous execution - if (dataset != null && dataset.exists()) { + try { + attemptCreateTableAndTestInsert(bigquery, tmpTestDatasetId); + } finally { bigquery.delete(tmpTestDatasetId); } + } - final DatasetInfo datasetInfo = DatasetInfo.newBuilder(tmpTestDatasetId).setLocation(datasetLocation).build(); - bigquery.create(datasetInfo); - bigquery.delete(tmpTestDatasetId); + /** + * Method is used to create tmp table and make dummy record insert. It's used in Check() connection + * method to make sure that user has all required roles for upcoming data sync/migration. It also + * verifies if BigQuery project is billable, if not - later sync will fail as non-billable project + * has limitations with stream uploading and DML queries. More details may be found there: + * https://cloud.google.com/bigquery/docs/streaming-data-into-bigquery + * https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language + * + * @param bigquery - initialized bigquery client + * @param tmpTestDatasetId - dataset name where tmp table will be created + */ + private static void attemptCreateTableAndTestInsert(final BigQuery bigquery, final String tmpTestDatasetId) { + // Create dummy schema that will be used for tmp table creation + final Schema testTableSchema = Schema.of( + Field.of("id", StandardSQLTypeName.INT64), + Field.of("name", StandardSQLTypeName.STRING)); + + // Create tmp table to verify if user has a create table permission. Also below we will do test + // records insert in it + final Table test_connection_table_name = createTable(bigquery, tmpTestDatasetId, + CHECK_TEST_TMP_TABLE_NAME, testTableSchema); + + // Try to make test (dummy records) insert to make sure that user has required permissions + try { + final InsertAllResponse response = + bigquery.insertAll(InsertAllRequest + .newBuilder(test_connection_table_name) + .addRow(Map.of("id", 1, "name", "James")) + .addRow(Map.of("id", 2, "name", "Eugene")) + .addRow(Map.of("id", 3, "name", "Angelina")) + .build()); + + if (response.hasErrors()) { + // If any of the insertions failed, this lets you inspect the errors + for (Map.Entry> entry : response.getInsertErrors().entrySet()) { + throw new ConfigErrorException("Failed to check connection: \n" + entry.getValue()); + } + } + } catch (final BigQueryException e) { + throw new ConfigErrorException("Failed to check connection: \n" + e.getMessage()); + } finally { + test_connection_table_name.delete(); + } + } + + public static Table createTable(final BigQuery bigquery, String datasetName, String tableName, Schema schema) { + final TableId tableId = TableId.of(datasetName, tableName); + final TableDefinition tableDefinition = StandardTableDefinition.of(schema); + final TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); + return bigquery.create(tableInfo); } // https://cloud.google.com/bigquery/docs/creating-partitioned-tables#java diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index f81de9d4fad3..4dae83c587ea 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -76,6 +76,8 @@ class BigQueryDestinationTest { protected static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); protected static final Path CREDENTIALS_WITH_MISSED_CREATE_DATASET_ROLE_PATH = Path.of("secrets/credentials-with-missed-dataset-creation-role.json"); + protected static final Path CREDENTIALS_NON_BILLABLE_PROJECT_PATH = + Path.of("secrets/credentials-non-billable-project.json"); private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestinationTest.class); private static final String DATASET_NAME_PREFIX = "bq_dest_integration_test"; @@ -250,8 +252,7 @@ void testCheckFailureInsufficientPermissionForCreateDataset(final DatasetIdReset please add file with creds to ../destination-bigquery/secrets/credentialsWithMissedDatasetCreationRole.json."""); } - final String fullConfigAsString = Files.readString( - CREDENTIALS_WITH_MISSED_CREATE_DATASET_ROLE_PATH); + final String fullConfigAsString = Files.readString(CREDENTIALS_WITH_MISSED_CREATE_DATASET_ROLE_PATH); final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString).get(BigQueryConsts.BIGQUERY_BASIC_CONFIG); final String projectId = credentialsJson.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); final String datasetId = Strings.addRandomSuffix(DATASET_NAME_PREFIX, "_", 8); @@ -276,6 +277,41 @@ void testCheckFailureInsufficientPermissionForCreateDataset(final DatasetIdReset assertThat(ex.getMessage()).contains("User does not have bigquery.datasets.create permission"); } + @ParameterizedTest + @MethodSource("datasetIdResetterProvider") + void testCheckFailureNonBillableProject(final DatasetIdResetter resetDatasetId) throws IOException { + + if (!Files.exists(CREDENTIALS_NON_BILLABLE_PROJECT_PATH)) { + throw new IllegalStateException(""" + Json config not found. Must provide path to a big query credentials file, + please add file with creds to + ../destination-bigquery/secrets/credentials-non-billable-project.json"""); + } + final String fullConfigAsString = Files.readString(CREDENTIALS_NON_BILLABLE_PROJECT_PATH); + + final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString).get(BigQueryConsts.BIGQUERY_BASIC_CONFIG); + final String projectId = credentialsJson.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); + + final JsonNode insufficientRoleConfig; + + insufficientRoleConfig = Jsons.jsonNode(ImmutableMap.builder() + .put(BigQueryConsts.CONFIG_PROJECT_ID, projectId) + .put(BigQueryConsts.CONFIG_CREDS, credentialsJson.toString()) + .put(BigQueryConsts.CONFIG_DATASET_ID, "testnobilling") + .put(BigQueryConsts.CONFIG_DATASET_LOCATION, "US") + .put(BIG_QUERY_CLIENT_CHUNK_SIZE, 10) + .build()); + + resetDatasetId.accept(insufficientRoleConfig); + + // Assert that check throws exception. Later it will be handled by IntegrationRunner + final ConfigErrorException ex = assertThrows(ConfigErrorException.class, () -> { + new BigQueryDestination().check(insufficientRoleConfig); + }); + + assertThat(ex.getMessage()).contains("Access Denied: BigQuery BigQuery: Streaming insert is not allowed in the free tier"); + } + @ParameterizedTest @MethodSource("datasetIdResetterProvider") void testWriteSuccess(final DatasetIdResetter resetDatasetId) throws Exception { diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 741bded1ba21..491e288f48d3 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -17,8 +17,8 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; -import io.airbyte.integrations.standardtest.destination.DataTypeTestArgumentProvider; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 60c9f5b43206..e96a6fdc075a 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -17,8 +17,8 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.ExtendedNameTransformer; -import io.airbyte.integrations.standardtest.destination.DataTypeTestArgumentProvider; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index a76e5983f8e1..809d8282e1ef 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -16,8 +16,8 @@ import io.airbyte.integrations.base.ssh.SshBastionContainer; import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.destination.ExtendedNameTransformer; -import io.airbyte.integrations.standardtest.destination.DataTypeTestArgumentProvider; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.ArrayList; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java index 89871b7b9b07..f39a963696a3 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java @@ -11,7 +11,7 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.destination.s3.S3Format; import io.airbyte.integrations.destination.s3.avro.JsonSchemaType; -import io.airbyte.integrations.standardtest.destination.NumberDataTypeTestArgumentProvider; +import io.airbyte.integrations.standardtest.destination.argproviders.NumberDataTypeTestArgumentProvider; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; diff --git a/airbyte-integrations/connectors/destination-scaffold-destination-python/setup.py b/airbyte-integrations/connectors/destination-scaffold-destination-python/setup.py index 28ce3e24d610..9acb10c22ab0 100644 --- a/airbyte-integrations/connectors/destination-scaffold-destination-python/setup.py +++ b/airbyte-integrations/connectors/destination-scaffold-destination-python/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = ["pytest~=6.1"] +TEST_REQUIREMENTS = ["pytest~=6.2"] setup( name="destination_scaffold_destination_python", diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java index 8e4367bc3e85..0829750a97fe 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java @@ -21,8 +21,8 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.standardtest.destination.DataArgumentsProvider; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteConnectionStatus; @@ -151,7 +151,7 @@ private List retrieveRecordsFromTable(final String tableName, final St return database.bufferedResultSetQuery( connection -> { try (final ResultSet tableInfo = connection.createStatement() - .executeQuery(String.format("SHOW TABLES LIKE '%s' IN SCHEMA %s;", tableName, schema));) { + .executeQuery(String.format("SHOW TABLES LIKE '%s' IN SCHEMA %s;", tableName, schema))) { assertTrue(tableInfo.next()); // check that we're creating permanent tables. DBT defaults to transient tables, which have // `TRANSIENT` as the value for the `kind` column. @@ -237,7 +237,7 @@ public void testSyncWithBillionRecords(final String messagesFilename, final Stri final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); final List messages = MoreResources.readResource(messagesFilename).lines() - .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); + .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).toList(); final List largeNumberRecords = Collections.nCopies(15000000, messages).stream().flatMap(List::stream).collect(Collectors.toList()); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java index 609882a6a59d..e73187ed00a8 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java @@ -9,7 +9,7 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.destination.DataArgumentsProvider; +import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; diff --git a/airbyte-integrations/connectors/source-asana/Dockerfile b/airbyte-integrations/connectors/source-asana/Dockerfile index e7dfbe4031ef..d2fd7c5765d7 100644 --- a/airbyte-integrations/connectors/source-asana/Dockerfile +++ b/airbyte-integrations/connectors/source-asana/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-asana diff --git a/airbyte-integrations/connectors/source-asana/README.md b/airbyte-integrations/connectors/source-asana/README.md index f0fcff4efa39..3e60ddf3edeb 100644 --- a/airbyte-integrations/connectors/source-asana/README.md +++ b/airbyte-integrations/connectors/source-asana/README.md @@ -101,7 +101,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. To run your integration tests with acceptance tests, from the connector root, run ``` -python -m pytest integration_tests -p integration_tests.acceptance +docker build . --no-cache -t airbyte/source-asana:dev \ +&& python -m pytest -p source_acceptance_test.plugin ``` To run your integration tests with docker diff --git a/airbyte-integrations/connectors/source-asana/acceptance-test-config.yml b/airbyte-integrations/connectors/source-asana/acceptance-test-config.yml index b919a29e843f..bf1454c6d836 100644 --- a/airbyte-integrations/connectors/source-asana/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-asana/acceptance-test-config.yml @@ -1,21 +1,34 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-asana:dev -tests: +test_strictness_level: high +acceptance_tests: spec: - - spec_path: "source_asana/spec.json" + tests: + - spec_path: "source_asana/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 7200 + tests: + - config_path: "secrets/config.json" + timeout_seconds: 7200 + expect_records: + bypass_reason: "Bypassed until dedicated sandbox account is up and running. Please follow https://github.com/airbytehq/airbyte/issues/19662." + empty_streams: + - name: custom_fields + bypass_reason: "This stream is not available on the account we're currently using. Please follow https://github.com/airbytehq/airbyte/issues/19662." full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 7200 + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # timeout_seconds: 7200 + bypass_reason: "As we are using an internal account the data is not frozen and results of `two-sequential-reads` are flaky. Please follow https://github.com/airbytehq/airbyte/issues/19662." + incremental: + bypass_reason: "Incremental syncs are not supported on this connector." diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index af1e11549f3b..06bcdc7d356f 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -2,23 +2,41 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from __future__ import annotations from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Type +from typing import Any, Iterable, Mapping, MutableMapping, Optional, Type import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +ASANA_ERRORS_MAPPING = { + 402: "This stream is available to premium organizations and workspaces only", + 403: "Missing permissions to consume this stream enough permissions", + 404: "The object specified by the request does not exist", + 451: "This request was blocked for legal reasons", +} + class AsanaStream(HttpStream, ABC): url_base = "https://app.asana.com/api/1.0/" - primary_key = "gid" - # Asana pagination could be from 1 to 100. page_size = 100 + raise_on_http_errors = True + + @property + def AsanaStreamType(self) -> Type: + return self.__class__ + + def should_retry(self, response: requests.Response) -> bool: + if response.status_code in ASANA_ERRORS_MAPPING.keys(): + self.logger.error( + f"Skipping stream {self.name}. {ASANA_ERRORS_MAPPING.get(response.status_code)}. Full error message: {response.text}" + ) + setattr(self, "raise_on_http_errors", False) + return False + return super().should_retry(response) def backoff_time(self, response: requests.Response) -> Optional[int]: delay_time = response.headers.get("Retry-After") @@ -31,17 +49,11 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, if next_page: return {"offset": next_page["offset"]} - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: params = {"limit": self.page_size} - params.update(self.get_opt_fields()) - if next_page_token: params.update(next_page_token) - return params def get_opt_fields(self) -> MutableMapping[str, str]: @@ -81,7 +93,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp response_json = response.json() yield from response_json.get("data", []) # Asana puts records in a container array "data" - def read_slices_from_records(self, stream_class: Type[AsanaStream], slice_field: str) -> Iterable[Optional[Mapping[str, Any]]]: + def read_slices_from_records(self, stream_class: AsanaStreamType, slice_field: str) -> Iterable[Optional[Mapping[str, Any]]]: """ General function for getting parent stream (which should be passed through `stream_class`) slice. Generates dicts with `gid` of parent streams. @@ -100,9 +112,7 @@ class WorkspaceRelatedStream(AsanaStream, ABC): into the path or will pass it as a request parameter. """ - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: workspaces_stream = Workspaces(authenticator=self.authenticator) for workspace in workspaces_stream.read_records(sync_mode=SyncMode.full_refresh): yield {"workspace_gid": workspace["gid"]} @@ -114,10 +124,8 @@ class WorkspaceRequestParamsRelatedStream(WorkspaceRelatedStream, ABC): So this is basically the whole point of this class - to pass `workspace` as request argument. """ - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) params["workspace"] = stream_slice["workspace_gid"] return params @@ -128,9 +136,7 @@ class ProjectRelatedStream(AsanaStream, ABC): argument in request. """ - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield from self.read_slices_from_records(stream_class=Projects, slice_field="project_gid") @@ -158,9 +164,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: task_gid = stream_slice["task_gid"] return f"tasks/{task_gid}/stories" - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield from self.read_slices_from_records(stream_class=Tasks, slice_field="task_gid") @@ -173,10 +177,8 @@ class Tasks(ProjectRelatedStream): def path(self, **kwargs) -> str: return "tasks" - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_slice=stream_slice, **kwargs) params["project"] = stream_slice["project_gid"] return params @@ -202,9 +204,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: team_gid = stream_slice["team_gid"] return f"teams/{team_gid}/team_memberships" - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield from self.read_slices_from_records(stream_class=Teams, slice_field="team_gid") diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py index 35c88a8cb312..6ddab5adeb1a 100644 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py @@ -30,3 +30,24 @@ def test_next_page_token(): inputs = {"response": MagicMock()} expected = "offset" assert expected in stream.next_page_token(**inputs) + + +@pytest.mark.parametrize( + ("http_status_code", "should_retry"), + [ + (402, False), + (403, False), + (404, False), + (451, False), + (429, True), + ], +) +def test_should_retry(http_status_code, should_retry): + """ + 402, 403, 404, 451 - should not retry. + 429 - should retry. + """ + response_mock = MagicMock() + response_mock.status_code = http_status_code + stream = Stories(MagicMock()) + assert stream.should_retry(response_mock) == should_retry diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 5628c15c53eb..7d6b21abe5a5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.72 +LABEL io.airbyte.version=0.2.74 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index f27ad1bfc415..a500163eb188 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -250,7 +250,7 @@ "action_breakdowns": { "title": "Action Breakdowns", "description": "A list of chosen action_breakdowns for action_breakdowns", - "default": [], + "default": ["action_type", "action_target_id", "action_destination"], "type": "array", "items": { "title": "ValidActionBreakdowns", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py index ac757a2d80ae..c25b0c89972a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/setup.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.9", "cached_property==1.5.2", "facebook_business==15.0.0", "pendulum>=2,<3", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 76ac52e64c7e..04b116a02a07 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -5,13 +5,14 @@ import logging from typing import Any, List, Mapping, Optional, Tuple, Type +import facebook_business import pendulum import requests from airbyte_cdk.models import AuthSpecification, ConnectorSpecification, DestinationSyncMode, OAuth2Specification from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from source_facebook_marketing.api import API -from source_facebook_marketing.spec import ConnectorConfig, InsightConfig +from source_facebook_marketing.spec import ConnectorConfig from source_facebook_marketing.streams import ( Activities, AdAccount, @@ -59,10 +60,17 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> try: api = API(account_id=config.account_id, access_token=config.access_token) logger.info(f"Select account {api.account}") - return True, None except requests.exceptions.RequestException as e: return False, e + # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters + for stream in self.get_custom_insights_streams(api, config): + try: + stream.check_breakdowns() + except facebook_business.exceptions.FacebookRequestError as e: + return False, e._api_error_message + return True, None + def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: """Discovery method, returns available streams @@ -149,7 +157,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: ), ] - return self._update_insights_streams(insights=config.custom_insights, default_args=insights_args, streams=streams) + return streams + self.get_custom_insights_streams(api, config) def spec(self, *args, **kwargs) -> ConnectorSpecification: """Returns the spec for this integration. @@ -170,28 +178,20 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: ), ) - def _update_insights_streams(self, insights: List[InsightConfig], default_args, streams) -> List[Type[Stream]]: - """Update method, if insights have values returns streams replacing the - default insights streams else returns streams - """ - if not insights: - return streams - - insights_custom_streams = list() - - for insight in insights: - args = dict( - api=default_args["api"], + def get_custom_insights_streams(self, api: API, config: ConnectorConfig) -> List[Type[Stream]]: + """return custom insights streams""" + streams = [] + for insight in config.custom_insights or []: + stream = AdsInsights( + api=api, name=f"Custom{insight.name}", fields=list(set(insight.fields)), breakdowns=list(set(insight.breakdowns)), action_breakdowns=list(set(insight.action_breakdowns)), time_increment=insight.time_increment, - start_date=insight.start_date or default_args["start_date"], - end_date=insight.end_date or default_args["end_date"], - insights_lookback_window=insight.insights_lookback_window or default_args["insights_lookback_window"], + start_date=insight.start_date or config.start_date, + end_date=insight.end_date or config.end_date, + insights_lookback_window=insight.insights_lookback_window or config.insights_lookback_window, ) - insight_stream = AdsInsights(**args) - insights_custom_streams.append(insight_stream) - - return streams + insights_custom_streams + streams.append(stream) + return streams diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 65497276005b..fc08360a99a1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -47,7 +47,7 @@ class Config: action_breakdowns: Optional[List[ValidActionBreakdowns]] = Field( title="Action Breakdowns", description="A list of chosen action_breakdowns for action_breakdowns", - default=[], + default=["action_type", "action_target_id", "action_destination"], ) time_increment: Optional[PositiveInt] = Field( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 55d4898f2b0c..bbb918d046b9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -49,13 +49,10 @@ class AdsInsights(FBMarketingIncrementalStream): # https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights/#overview INSIGHTS_RETENTION_PERIOD = pendulum.duration(months=37) - action_breakdowns = ALL_ACTION_BREAKDOWNS level = "ad" action_attribution_windows = ALL_ACTION_ATTRIBUTION_WINDOWS time_increment = 1 - breakdowns = [] - def __init__( self, name: str = None, @@ -70,8 +67,8 @@ def __init__( self._start_date = self._start_date.date() self._end_date = self._end_date.date() self._fields = fields - self.action_breakdowns = action_breakdowns or self.action_breakdowns - self.breakdowns = breakdowns or self.breakdowns + self.action_breakdowns = action_breakdowns or self.ALL_ACTION_BREAKDOWNS + self.breakdowns = breakdowns or [] self.time_increment = time_increment or self.time_increment self._new_class_name = name self._insights_lookback_window = insights_lookback_window @@ -198,6 +195,18 @@ def _generate_async_jobs(self, params: Mapping) -> Iterator[AsyncJob]: interval = pendulum.Period(ts_start, ts_end) yield InsightAsyncJob(api=self._api.api, edge_object=self._api.account, interval=interval, params=params) + def check_breakdowns(self): + """ + Making call to check "action_breakdowns" and "breakdowns" combinations + https://developers.facebook.com/docs/marketing-api/insights/breakdowns#combiningbreakdowns + """ + params = { + "action_breakdowns": self.action_breakdowns, + "breakdowns": self.breakdowns, + "fields": ["account_id"], + } + self._api.account.get_insights(params=params, is_async=False) + def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 3b0a7e3e8bf1..ac0352b2e445 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -99,20 +99,12 @@ def test_spec(self): assert isinstance(spec, ConnectorSpecification) - def test_update_insights_streams(self, api, config): + def test_get_custom_insights_streams(self, api, config): config["custom_insights"] = [ {"name": "test", "fields": ["account_id"], "breakdowns": ["ad_format_asset"], "action_breakdowns": ["action_device"]}, ] - streams = SourceFacebookMarketing().streams(config) config = ConnectorConfig.parse_obj(config) - insights_args = dict( - api=api, - start_date=config.start_date, - end_date=config.end_date, - ) - assert SourceFacebookMarketing()._update_insights_streams( - insights=config.custom_insights, default_args=insights_args, streams=streams - ) + assert SourceFacebookMarketing().get_custom_insights_streams(api, config) def test_check_config(config_gen, requests_mock): diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index 409d7e734fa8..d6b0a2a75460 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml index ffbd92b657d5..5d89c3502494 100644 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -1,34 +1,39 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-google-ads:dev -tests: +acceptance_tests: spec: - - spec_path: "source_google_ads/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.45" + tests: + - spec_path: "source_google_ads/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.45" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.2.4" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: - [ - "geographic_report", - "keyword_report", - "display_keyword_performance_report", - "display_topics_performance_report", - "shopping_performance_report", - "unhappytable", - "click_view", - ] - timeout_seconds: 600 - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_protobuf_msg.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "geographic_report" + - name: "keyword_report" + - name: "display_keyword_performance_report" + - name: "display_topics_performance_report" + - name: "shopping_performance_report" + - name: "unhappytable" + - name: "click_view" + timeout_seconds: 600 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_protobuf_msg.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json index 99c9e0d1013a..c4f354d79ce7 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json @@ -249,10 +249,10 @@ "type": ["null", "number"] }, "metrics.conversions": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "metrics.conversions_value": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "metrics.cost_micros": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index 367d4518cbac..f432e604e8d6 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.13 +LABEL io.airbyte.version=1.0.14 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 6bdb2fa01373..803a8528b87d 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.13 +LABEL io.airbyte.version=1.0.14 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 2ce87b9be8f5..1af75b8a6d1d 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.25 +LABEL io.airbyte.version=1.0.28 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 7dfbacc0d6ba..6d48544ae92b 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.25 +LABEL io.airbyte.version=1.0.28 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 321c075f9e92..bfb424984740 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -58,7 +58,6 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.sql.Connection; -import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.Duration; @@ -74,7 +73,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PostgresSource extends AbstractJdbcSource implements Source { +public class PostgresSource extends AbstractJdbcSource implements Source { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresSource.class); private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; @@ -91,9 +90,6 @@ public class PostgresSource extends AbstractJdbcSource implements Sour public static final String SSL_KEY = "sslkey"; public static final String SSL_PASSWORD = "sslpassword"; public static final String MODE = "mode"; - static final Map SSL_JDBC_PARAMETERS = ImmutableMap.of( - "ssl", "true", - "sslmode", "require"); private List schemas; private final FeatureFlags featureFlags; private static final Set INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer"); @@ -109,11 +105,7 @@ public static Source sshWrappedSource() { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - if (JdbcUtils.useSsl(config)) { - return SSL_JDBC_PARAMETERS; - } else { - return Collections.emptyMap(); - } + return Collections.emptyMap(); } @Override @@ -174,7 +166,7 @@ public String toJDBCQueryParams(final Map sslParams) { .map((entry) -> { try { final String result = switch (entry.getKey()) { - case SSL_MODE -> PARAM_SSLMODE + EQUALS + toSslJdbcParam(SslMode.valueOf(entry.getValue())) + case AbstractJdbcSource.SSL_MODE -> PARAM_SSLMODE + EQUALS + toSslJdbcParam(SslMode.valueOf(entry.getValue())) + JdbcUtils.AMPERSAND + PARAM_SSL + EQUALS + (entry.getValue() == DISABLE ? PARAM_SSL_FALSE : PARAM_SSL_TRUE); case CA_CERTIFICATE_PATH -> SSL_ROOT_CERT + EQUALS + entry.getValue(); case CLIENT_KEY_STORE_URL -> SSL_KEY + EQUALS + Path.of(new URI(entry.getValue())); @@ -214,8 +206,8 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { } @Override - public List>> discoverInternal(final JdbcDatabase database) throws Exception { - final List>> rawTables = discoverRawTables(database); + public List>> discoverInternal(final JdbcDatabase database) throws Exception { + final List>> rawTables = discoverRawTables(database); final Set publicizedTablesInCdc = PostgresCdcCatalogHelper.getPublicizedTables(database); if (publicizedTablesInCdc.isEmpty()) { @@ -227,15 +219,15 @@ public List>> discoverInternal(final JdbcDatabas .collect(toList()); } - public List>> discoverRawTables(final JdbcDatabase database) throws Exception { + public List>> discoverRawTables(final JdbcDatabase database) throws Exception { if (schemas != null && !schemas.isEmpty()) { // process explicitly selected (from UI) schemas - final List>> internals = new ArrayList<>(); + final List>> internals = new ArrayList<>(); for (final String schema : schemas) { LOGGER.info("Checking schema: {}", schema); - final List>> tables = super.discoverInternal(database, schema); + final List>> tables = super.discoverInternal(database, schema); internals.addAll(tables); - for (final TableInfo> table : tables) { + for (final TableInfo> table : tables) { LOGGER.info("Found table: {}.{}", table.getNameSpace(), table.getName()); } } @@ -325,7 +317,7 @@ public AutoCloseableIterator read(final JsonNode config, @Override public List> getIncrementalIterators(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, - final Map>> tableNameToTable, + final Map>> tableNameToTable, final StateManager stateManager, final Instant emittedAt) { final JsonNode sourceConfig = database.getSourceConfig(); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index c4e6d1b46804..7c51ee51cecb 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -4,25 +4,33 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; +import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; +import static io.airbyte.integrations.source.postgres.PostgresType.safeGetJdbcType; +import static io.airbyte.protocol.models.JsonSchemaType.INTEGER; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BinaryNode; +import com.fasterxml.jackson.databind.node.NullNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.db.DataTypeUtils; +import io.airbyte.db.SourceOperations; +import io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations; import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.JsonSchemaType; import java.math.BigDecimal; -import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -46,7 +54,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PostgresSourceOperations extends JdbcSourceOperations { +public class PostgresSourceOperations extends AbstractJdbcCompatibleSourceOperations + implements SourceOperations { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresSourceOperations.class); private static final String TIMESTAMPTZ = "timestamptz"; @@ -92,7 +101,7 @@ public JsonNode rowToJson(final ResultSet queryContext) throws SQLException { @Override public void setStatementField(final PreparedStatement preparedStatement, final int parameterIndex, - final JDBCType cursorFieldType, + final PostgresType cursorFieldType, final String value) throws SQLException { switch (cursorFieldType) { @@ -170,7 +179,7 @@ public void setJsonField(final ResultSet resultSet, final int colIndex, final Ob final PgResultSetMetaData metadata = (PgResultSetMetaData) resultSet.getMetaData(); final String columnName = metadata.getColumnName(colIndex); final String columnTypeName = metadata.getColumnTypeName(colIndex).toLowerCase(); - final JDBCType columnType = safeGetJdbcType(metadata.getColumnType(colIndex)); + final PostgresType columnType = safeGetJdbcType(metadata.getColumnType(colIndex)); if (resultSet.getString(colIndex) == null) { json.putNull(columnName); } else { @@ -188,6 +197,19 @@ public void setJsonField(final ResultSet resultSet, final int colIndex, final Ob case "path" -> putObject(json, columnName, resultSet, colIndex, PGpath.class); case "point" -> putObject(json, columnName, resultSet, colIndex, PGpoint.class); case "polygon" -> putObject(json, columnName, resultSet, colIndex, PGpolygon.class); + case "_varchar", "_char", "_bpchar", "_text", "_name" -> putArray(json, columnName, resultSet, colIndex); + case "_int2", "_int4", "_int8", "_oid" -> putLongArray(json, columnName, resultSet, colIndex); + case "_numeric", "_decimal" -> putBigDecimalArray(json, columnName, resultSet, colIndex); + case "_money" -> putMoneyArray(json, columnName, resultSet, colIndex); + case "_float4", "_float8" -> putDoubleArray(json, columnName, resultSet, colIndex); + case "_bool" -> putBooleanArray(json, columnName, resultSet, colIndex); + case "_bit" -> putBitArray(json, columnName, resultSet, colIndex); + case "_bytea" -> putByteaArray(json, columnName, resultSet, colIndex); + case "_date" -> putDateArray(json, columnName, resultSet, colIndex); + case "_timestamptz" -> putTimestampTzArray(json, columnName, resultSet, colIndex); + case "_timestamp" -> putTimestampArray(json, columnName, resultSet, colIndex); + case "_timetz" -> putTimeTzArray(json, columnName, resultSet, colIndex); + case "_time" -> putTimeArray(json, columnName, resultSet, colIndex); default -> { switch (columnType) { case BOOLEAN -> putBoolean(json, columnName, resultSet, colIndex); @@ -211,6 +233,156 @@ public void setJsonField(final ResultSet resultSet, final int colIndex, final Ob } } + private void putTimeArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final LocalTime time = getObject(arrayResultSet, 2, LocalTime.class); + if (time == null) { + arrayNode.add(NullNode.getInstance()); + } else { + arrayNode.add(DateTimeConverter.convertToTime(time)); + } + } + node.set(columnName, arrayNode); + } + + private void putTimeTzArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final OffsetTime timetz = getObject(arrayResultSet, 2, OffsetTime.class); + if (timetz == null) { + arrayNode.add(NullNode.getInstance()); + } else { + arrayNode.add(timetz.format(TIMETZ_FORMATTER)); + } + } + node.set(columnName, arrayNode); + } + + private void putTimestampArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final LocalDateTime timestamp = getObject(arrayResultSet, 2, LocalDateTime.class); + if (timestamp == null) { + arrayNode.add(NullNode.getInstance()); + } else { + arrayNode.add(DateTimeConverter.convertToTimestamp(timestamp)); + } + } + node.set(columnName, arrayNode); + } + + private void putTimestampTzArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final OffsetDateTime timestamptz = getObject(arrayResultSet, 2, OffsetDateTime.class); + if (timestamptz == null) { + arrayNode.add(NullNode.getInstance()); + } else { + final LocalDate localDate = timestamptz.toLocalDate(); + arrayNode.add(resolveEra(localDate, timestamptz.format(TIMESTAMPTZ_FORMATTER))); + } + } + node.set(columnName, arrayNode); + } + + private void putDateArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final LocalDate date = getObject(arrayResultSet, 2, LocalDate.class); + if (date == null) { + arrayNode.add(NullNode.getInstance()); + } else { + arrayNode.add(DateTimeConverter.convertToDate(date)); + } + } + node.set(columnName, arrayNode); + } + + private void putByteaArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + arrayNode.add(new BinaryNode(arrayResultSet.getBytes(2))); + } + node.set(columnName, arrayNode); + } + + private void putBitArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final String res = arrayResultSet.getString(2); + if (res == null) { + arrayNode.add(NullNode.getInstance()); + } else { + arrayNode.add("1".equals(res)); + } + } + node.set(columnName, arrayNode); + } + + private void putBooleanArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final String res = arrayResultSet.getString(2); + if (res == null) { + arrayNode.add(NullNode.getInstance()); + } else { + arrayNode.add("t".equalsIgnoreCase(res)); + } + } + node.set(columnName, arrayNode); + } + + private void putBigDecimalArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final BigDecimal bigDecimal = DataTypeUtils.returnNullIfInvalid(() -> arrayResultSet.getBigDecimal(2)); + if (bigDecimal != null) { + arrayNode.add(bigDecimal); + } else { + arrayNode.add((BigDecimal) null); + } + } + node.set(columnName, arrayNode); + } + + private void putDoubleArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + arrayNode.add(DataTypeUtils.returnNullIfInvalid(() -> arrayResultSet.getDouble(colIndex), Double::isFinite)); + } + node.set(columnName, arrayNode); + } + + private void putMoneyArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + final String moneyValue = parseMoneyValue(arrayResultSet.getString(colIndex)); + arrayNode.add(DataTypeUtils.returnNullIfInvalid(() -> DataTypeUtils.returnNullIfInvalid(() -> Double.valueOf(moneyValue), Double::isFinite))); + } + node.set(columnName, arrayNode); + } + + private void putLongArray(final ObjectNode node, final String columnName, final ResultSet resultSet, final int colIndex) throws SQLException { + final ArrayNode arrayNode = Jsons.arrayNode(); + final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); + while (arrayResultSet.next()) { + arrayNode.add(arrayResultSet.getLong(2)); + } + node.set(columnName, arrayNode); + } + @Override protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, LocalDate.class))); @@ -227,19 +399,41 @@ protected void putTimestamp(final ObjectNode node, final String columnName, fina } @Override - public JDBCType getFieldType(final JsonNode field) { + public PostgresType getFieldType(final JsonNode field) { try { final String typeName = field.get(INTERNAL_COLUMN_TYPE_NAME).asText().toLowerCase(); // Postgres boolean is mapped to JDBCType.BIT, but should be BOOLEAN return switch (typeName) { - case "bool", "boolean" -> JDBCType.BOOLEAN; + + case "_bit" -> PostgresType.BIT_ARRAY; + case "_bool" -> PostgresType.BOOL_ARRAY; + case "_name" -> PostgresType.NAME_ARRAY; + case "_varchar" -> PostgresType.VARCHAR_ARRAY; + case "_char" -> PostgresType.CHAR_ARRAY; + case "_bpchar" -> PostgresType.BPCHAR_ARRAY; + case "_text" -> PostgresType.TEXT_ARRAY; + case "_int4" -> PostgresType.INT4_ARRAY; + case "_int2" -> PostgresType.INT2_ARRAY; + case "_int8" -> PostgresType.INT8_ARRAY; + case "_money" -> PostgresType.MONEY_ARRAY; + case "_oid" -> PostgresType.OID_ARRAY; + case "_numeric" -> PostgresType.NUMERIC_ARRAY; + case "_float4" -> PostgresType.FLOAT4_ARRAY; + case "_float8" -> PostgresType.FLOAT8_ARRAY; + case "_timestamptz" -> PostgresType.TIMESTAMPTZ_ARRAY; + case "_timestamp" -> PostgresType.TIMESTAMP_ARRAY; + case "_timetz" -> PostgresType.TIMETZ_ARRAY; + case "_time" -> PostgresType.TIME_ARRAY; + case "_date" -> PostgresType.DATE_ARRAY; + case "_bytea" -> PostgresType.BYTEA_ARRAY; + case "bool", "boolean" -> PostgresType.BOOLEAN; // BYTEA is variable length binary string with hex output format by default (e.g. "\x6b707a"). // It should not be converted to base64 binary string. So it is represented as JDBC VARCHAR. // https://www.postgresql.org/docs/14/datatype-binary.html - case "bytea" -> JDBCType.VARCHAR; - case TIMESTAMPTZ -> JDBCType.TIMESTAMP_WITH_TIMEZONE; - case TIMETZ -> JDBCType.TIME_WITH_TIMEZONE; - default -> JDBCType.valueOf(field.get(INTERNAL_COLUMN_TYPE).asInt()); + case "bytea" -> PostgresType.VARCHAR; + case TIMESTAMPTZ -> PostgresType.TIMESTAMP_WITH_TIMEZONE; + case TIMETZ -> PostgresType.TIME_WITH_TIMEZONE; + default -> PostgresType.valueOf(field.get(INTERNAL_COLUMN_TYPE).asInt()); }; } catch (final IllegalArgumentException ex) { LOGGER.warn(String.format("Could not convert column: %s from table: %s.%s with type: %s. Casting to VARCHAR.", @@ -247,24 +441,100 @@ public JDBCType getFieldType(final JsonNode field) { field.get(INTERNAL_SCHEMA_NAME), field.get(INTERNAL_TABLE_NAME), field.get(INTERNAL_COLUMN_TYPE))); - return JDBCType.VARCHAR; + return PostgresType.VARCHAR; } } @Override - public JsonSchemaType getJsonType(final JDBCType jdbcType) { + public JsonSchemaType getJsonType(final PostgresType jdbcType) { return switch (jdbcType) { case BOOLEAN -> JsonSchemaType.BOOLEAN; case TINYINT, SMALLINT, INTEGER, BIGINT -> JsonSchemaType.INTEGER; case FLOAT, DOUBLE, REAL, NUMERIC, DECIMAL -> JsonSchemaType.NUMBER; case BLOB, BINARY, VARBINARY, LONGVARBINARY -> JsonSchemaType.STRING_BASE_64; case ARRAY -> JsonSchemaType.ARRAY; + case BIT_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.BOOLEAN) + .build()) + .build(); + case BOOL_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.BOOLEAN) + .build()) + .build(); + case BYTEA_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build(); + case NAME_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build(); + case VARCHAR_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build(); + case CHAR_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build(); + case BPCHAR_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build(); + case TEXT_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build(); + case INT4_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.INTEGER) + .build(); + case INT2_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.INTEGER) + .build(); + case INT8_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.INTEGER) + .build(); + case MONEY_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build(); + case OID_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build(); + case NUMERIC_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build(); + case FLOAT4_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build(); + case FLOAT8_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build(); + case TIMESTAMPTZ_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE) + .build(); + case TIMESTAMP_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) + .build(); + case TIMETZ_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.STRING_TIME_WITH_TIMEZONE) + .build(); + case TIME_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE) + .build(); + case DATE_ARRAY -> JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.STRING_DATE) + .build(); + case DATE -> JsonSchemaType.STRING_DATE; case TIME -> JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE; case TIME_WITH_TIMEZONE -> JsonSchemaType.STRING_TIME_WITH_TIMEZONE; case TIMESTAMP -> JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE; case TIMESTAMP_WITH_TIMEZONE -> JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE; - default -> JsonSchemaType.STRING; }; } @@ -329,7 +599,7 @@ static String parseMoneyValue(final String moneyString) { } @Override - public boolean isCursorType(JDBCType type) { + public boolean isCursorType(PostgresType type) { return PostgresUtils.ALLOWED_CURSOR_TYPES.contains(type); } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresType.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresType.java new file mode 100644 index 000000000000..43d61299cf5f --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresType.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +import java.sql.SQLType; +import java.sql.Types; + +public enum PostgresType implements SQLType { + + BIT(Types.BIT), + TINYINT(Types.TINYINT), + SMALLINT(Types.SMALLINT), + INTEGER(Types.INTEGER), + BIGINT(Types.BIGINT), + FLOAT(Types.FLOAT), + REAL(Types.REAL), + DOUBLE(Types.DOUBLE), + NUMERIC(Types.NUMERIC), + DECIMAL(Types.DECIMAL), + CHAR(Types.CHAR), + VARCHAR(Types.VARCHAR), + LONGVARCHAR(Types.LONGVARCHAR), + DATE(Types.DATE), + TIME(Types.TIME), + TIMESTAMP(Types.TIMESTAMP), + BINARY(Types.BINARY), + VARBINARY(Types.VARBINARY), + LONGVARBINARY(Types.LONGVARBINARY), + NULL(Types.NULL), + OTHER(Types.OTHER), + JAVA_OBJECT(Types.JAVA_OBJECT), + DISTINCT(Types.DISTINCT), + STRUCT(Types.STRUCT), + ARRAY(Types.ARRAY), + BLOB(Types.BLOB), + CLOB(Types.CLOB), + REF(Types.REF), + DATALINK(Types.DATALINK), + BOOLEAN(Types.BOOLEAN), + ROWID(Types.ROWID), + NCHAR(Types.NCHAR), + NVARCHAR(Types.NVARCHAR), + LONGNVARCHAR(Types.LONGNVARCHAR), + NCLOB(Types.NCLOB), + SQLXML(Types.SQLXML), + REF_CURSOR(Types.REF_CURSOR), + TIME_WITH_TIMEZONE(Types.TIME_WITH_TIMEZONE), + TIMESTAMP_WITH_TIMEZONE(Types.TIMESTAMP_WITH_TIMEZONE), + VARCHAR_ARRAY(Types.ARRAY), + TEXT_ARRAY(Types.ARRAY), + INTEGER_ARRAY(Types.ARRAY), + NUMERIC_ARRAY(Types.ARRAY), + TIMESTAMPTZ_ARRAY(Types.ARRAY), + TIMESTAMP_ARRAY(Types.ARRAY), + TIMETZ_ARRAY(Types.ARRAY), + TIME_ARRAY(Types.ARRAY), + DATE_ARRAY(Types.ARRAY), + BIT_ARRAY(Types.ARRAY), + BOOL_ARRAY(Types.ARRAY), + NAME_ARRAY(Types.ARRAY), + CHAR_ARRAY(Types.ARRAY), + BPCHAR_ARRAY(Types.ARRAY), + INT4_ARRAY(Types.ARRAY), + INT2_ARRAY(Types.ARRAY), + INT8_ARRAY(Types.ARRAY), + MONEY_ARRAY(Types.ARRAY), + OID_ARRAY(Types.ARRAY), + FLOAT4_ARRAY(Types.ARRAY), + FLOAT8_ARRAY(Types.ARRAY), + BYTEA_ARRAY(Types.ARRAY); + + /** + * The Integer value for the JDBCType. It maps to a value in {@code Types.java} + */ + private Integer type; + + /** + * Constructor to specify the data type value from {@code Types) for + * this data type. @param type The value from {@code Types) for this data type + */ + PostgresType(final Integer type) { + this.type = type; + } + + /** + * {@inheritDoc } + * + * @return The name of this {@code SQLType}. + */ + public String getName() { + return name(); + } + + /** + * Returns the name of the vendor that supports this data type. + * + * @return The name of the vendor for this data type which is {@literal java.sql} for JDBCType. + */ + public String getVendor() { + return "java.sql"; + } + + /** + * Returns the vendor specific type number for the data type. + * + * @return An Integer representing the data type. For {@code JDBCType}, the value will be the same + * value as in {@code Types} for the data type. + */ + public Integer getVendorTypeNumber() { + return type; + } + + /** + * Returns the {@code JDBCType} that corresponds to the specified {@code Types} value + * + * @param type {@code Types} value + * @return The {@code JDBCType} constant + * @throws IllegalArgumentException if this enum type has no constant with the specified + * {@code Types} value + * @see Types + */ + public static PostgresType valueOf(int type) { + for (PostgresType sqlType : PostgresType.class.getEnumConstants()) { + if (type == sqlType.type) + return sqlType; + } + throw new IllegalArgumentException("Type:" + type + " is not a valid " + + "Types.java value."); + } + + public static PostgresType safeGetJdbcType(final int columnTypeInt) { + try { + return PostgresType.valueOf(columnTypeInt); + } catch (final Exception e) { + return PostgresType.VARCHAR; + } + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java index 69396230012c..ebc2f1f9870b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java @@ -4,26 +4,25 @@ package io.airbyte.integrations.source.postgres; -import static java.sql.JDBCType.BIGINT; -import static java.sql.JDBCType.DATE; -import static java.sql.JDBCType.DECIMAL; -import static java.sql.JDBCType.DOUBLE; -import static java.sql.JDBCType.FLOAT; -import static java.sql.JDBCType.INTEGER; -import static java.sql.JDBCType.LONGVARCHAR; -import static java.sql.JDBCType.NUMERIC; -import static java.sql.JDBCType.NVARCHAR; -import static java.sql.JDBCType.REAL; -import static java.sql.JDBCType.SMALLINT; -import static java.sql.JDBCType.TIME; -import static java.sql.JDBCType.TIMESTAMP; -import static java.sql.JDBCType.TIMESTAMP_WITH_TIMEZONE; -import static java.sql.JDBCType.TIME_WITH_TIMEZONE; -import static java.sql.JDBCType.TINYINT; -import static java.sql.JDBCType.VARCHAR; +import static io.airbyte.integrations.source.postgres.PostgresType.BIGINT; +import static io.airbyte.integrations.source.postgres.PostgresType.DATE; +import static io.airbyte.integrations.source.postgres.PostgresType.DECIMAL; +import static io.airbyte.integrations.source.postgres.PostgresType.DOUBLE; +import static io.airbyte.integrations.source.postgres.PostgresType.FLOAT; +import static io.airbyte.integrations.source.postgres.PostgresType.INTEGER; +import static io.airbyte.integrations.source.postgres.PostgresType.LONGVARCHAR; +import static io.airbyte.integrations.source.postgres.PostgresType.NUMERIC; +import static io.airbyte.integrations.source.postgres.PostgresType.NVARCHAR; +import static io.airbyte.integrations.source.postgres.PostgresType.REAL; +import static io.airbyte.integrations.source.postgres.PostgresType.SMALLINT; +import static io.airbyte.integrations.source.postgres.PostgresType.TIME; +import static io.airbyte.integrations.source.postgres.PostgresType.TIMESTAMP; +import static io.airbyte.integrations.source.postgres.PostgresType.TIMESTAMP_WITH_TIMEZONE; +import static io.airbyte.integrations.source.postgres.PostgresType.TIME_WITH_TIMEZONE; +import static io.airbyte.integrations.source.postgres.PostgresType.TINYINT; +import static io.airbyte.integrations.source.postgres.PostgresType.VARCHAR; import com.fasterxml.jackson.databind.JsonNode; -import java.sql.JDBCType; import java.time.Duration; import java.util.Optional; import java.util.Set; @@ -32,7 +31,7 @@ public class PostgresUtils { - public static final Set ALLOWED_CURSOR_TYPES = Set.of(TIMESTAMP, TIMESTAMP_WITH_TIMEZONE, TIME, TIME_WITH_TIMEZONE, + public static final Set ALLOWED_CURSOR_TYPES = Set.of(TIMESTAMP, TIMESTAMP_WITH_TIMEZONE, TIME, TIME_WITH_TIMEZONE, DATE, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, REAL, NUMERIC, DECIMAL, NVARCHAR, VARCHAR, LONGVARCHAR); private static final Logger LOGGER = LoggerFactory.getLogger(PostgresUtils.class); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json index 4dfdfab3fb6e..31c5acf4a36b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json @@ -240,7 +240,7 @@ "method": { "type": "string", "const": "CDC", - "order": 0 + "order": 1 }, "plugin": { "type": "string", @@ -248,26 +248,26 @@ "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], "const": "pgoutput", - "order": 1 + "order": 2 }, "replication_slot": { "type": "string", "title": "Replication Slot", "description": "A plugin logical replication slot. Read about replication slots.", - "order": 2 + "order": 3 }, "publication": { "type": "string", "title": "Publication", "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", - "order": 3 + "order": 4 }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", "default": 300, - "order": 4, + "order": 5, "min": 120, "max": 1200 } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java index a8d28976f59e..9faff3447958 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java @@ -4,9 +4,16 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static io.airbyte.protocol.models.JsonSchemaType.STRING_DATE; +import static io.airbyte.protocol.models.JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE; +import static io.airbyte.protocol.models.JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE; +import static io.airbyte.protocol.models.JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE; +import static io.airbyte.protocol.models.JsonSchemaType.STRING_TIME_WITH_TIMEZONE; + import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; import io.airbyte.integrations.standardtest.source.TestDataHolder; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.JsonSchemaType; import java.util.Set; import org.jooq.DSLContext; @@ -545,16 +552,6 @@ protected void initTests() { .addExpectedValues("(\"2010-01-01 14:30:00\",\"2010-01-01 15:30:00\")", null) .build()); - // array - addDataTypeTestData( - TestDataHolder.builder() - .sourceType("text") - .fullSourceDataType("text[]") - .airbyteType(JsonSchemaType.ARRAY) - .addInsertValues("'{10001, 10002, 10003, 10004}'", "null") - .addExpectedValues("[\"10001\",\"10002\",\"10003\",\"10004\"]", null) - .build()); - // composite type addDataTypeTestData( TestDataHolder.builder() @@ -580,6 +577,7 @@ protected void initTests() { .build()); addTimeWithTimeZoneTest(); + addArraysTestData(); } protected void addTimeWithTimeZoneTest() { @@ -617,4 +615,266 @@ protected void addTimestampWithInfinityValuesTest() { } } + private void addArraysTestData() { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("int2_array") + .fullSourceDataType("INT2[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.INTEGER) + .build()) + .addInsertValues("'{1,2,3}'", "'{4,5,6}'") + .addExpectedValues("[1,2,3]", "[4,5,6]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("int4_array") + .fullSourceDataType("INT4[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.INTEGER) + .build()) + .addInsertValues("'{-2147483648,2147483646}'") + .addExpectedValues("[-2147483648,2147483646]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("int8_array") + .fullSourceDataType("INT8[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.INTEGER) + .build()) + .addInsertValues("'{-9223372036854775808,9223372036854775801}'") + .addExpectedValues("[-9223372036854775808,9223372036854775801]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("oid_array") + .fullSourceDataType("OID[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build()) + .addInsertValues("'{564182,234181}'") + .addExpectedValues("[564182,234181]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("varchar_array") + .fullSourceDataType("VARCHAR[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build()) + .addInsertValues("'{lorem ipsum,dolor sit,amet}'") + .addExpectedValues("[\"lorem ipsum\",\"dolor sit\",\"amet\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("char_array") + .fullSourceDataType("CHAR(1)[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build()) + .addInsertValues("'{l,d,a}'") + .addExpectedValues("[\"l\",\"d\",\"a\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("bpchar_array") + .fullSourceDataType("BPCHAR(2)[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build()) + .addInsertValues("'{l,d,a}'") + .addExpectedValues("[\"l \",\"d \",\"a \"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("text_array") + .fullSourceDataType("TEXT[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build()) + .addInsertValues("'{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}'") + .addExpectedValues("[\"someeeeee loooooooooong teeeeext\",\"vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("name_array") + .fullSourceDataType("NAME[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build()) + .addInsertValues("'{object,integer}'") + .addExpectedValues("[\"object\",\"integer\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("numeric_array") + .fullSourceDataType("NUMERIC[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build()) + .addInsertValues("'{131070.23,231072.476596593}'") + .addExpectedValues("[131070.23,231072.476596593]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("decimal_array") + .fullSourceDataType("DECIMAL[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build()) + .addInsertValues("'{131070.23,231072.476596593}'") + .addExpectedValues("[131070.23,231072.476596593]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("float4_array") + .fullSourceDataType("FLOAT4[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build()) + .addInsertValues("'{131070.237689,231072.476596593}'") + .addExpectedValues("[131070.234,231072.48]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("float8_array") + .fullSourceDataType("FLOAT8[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build()) + .addInsertValues("'{131070.237689,231072.476596593}'") + .addExpectedValues("[131070.237689,231072.476596593]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("money_array") + .fullSourceDataType("MONEY[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER) + .build()) + .build()) + .addInsertValues("'{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}'") + .addExpectedValues("[999.99,1001.01,45000.0,1.0,800.0,22222.01,1001.01]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("bool_array") + .fullSourceDataType("BOOL[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.BOOLEAN) + .build()) + .build()) + .addInsertValues("'{true,yes,1,false,no,0,null}'") + .addExpectedValues("[true,true,true,false,false,false,null]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("bit_array") + .fullSourceDataType("BIT[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.BOOLEAN) + .build()) + .build()) + .addInsertValues("'{null,1,0}'") + .addExpectedValues("[null,true,false]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("bytea_array") + .fullSourceDataType("BYTEA[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .build()) + .build()) + .addInsertValues( + "'{\\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\\x4B52414B00000000000000000000000000000000000000000000000000000000}'") + .addExpectedValues( + "[\"eEE2Njk3RTk3NEU2QTMyMEY0NTQzOTBCRTAzRjc0OTU1RTg5NzhGMUE2OTcxRUE2NzMwNTQyRTM3QjY2MTc5QkM=\",\"eDRCNTI0MTRCMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("date_array") + .fullSourceDataType("DATE[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(STRING_DATE) + .build()) + .addInsertValues("'{1999-01-08,1991-02-10 BC}'") + .addExpectedValues("[\"1999-01-08\",\"1991-02-10 BC\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("time_array") + .fullSourceDataType("TIME(6)[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(STRING_TIME_WITHOUT_TIMEZONE) + .build()) + .addInsertValues("'{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}'") + .addExpectedValues( + "[\"13:00:01.000000\",\"13:00:02.000000\",\"13:00:03.000000\",\"13:00:04.000000\",\"13:00:05.000000\",\"13:00:00.000000\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("timetz_array") + .fullSourceDataType("TIMETZ[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(STRING_TIME_WITH_TIMEZONE) + .build()) + .addInsertValues("'{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}'") + .addExpectedValues( + "[null,\"13:00:01.000000-07:00\",\"13:00:00.000000+08:00\",\"13:00:03.000000-08:00\",\"13:00:04.000000Z\",\"13:00:05.012345-08:00\",\"13:00:06.000000+08:00\",\"13:00:00.000000-07:00\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("timestamptz_array") + .fullSourceDataType("TIMESTAMPTZ[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(STRING_TIMESTAMP_WITH_TIMEZONE) + .build()) + .addInsertValues("'{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}'") + .addExpectedValues("[null,\"2004-10-19T18:23:00.000000Z\",\"2004-10-19T18:23:54.123456Z\"]") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("timestamp_array") + .fullSourceDataType("TIMESTAMP[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(STRING_TIMESTAMP_WITHOUT_TIMEZONE) + .build()) + .addInsertValues("'{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'") + .addExpectedValues("[null,\"2004-10-19T10:23:00.000000\",\"2004-10-19T10:23:54.123456\",\"3004-10-19T10:23:54.123456 BC\"]") + .build()); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json index 5875205e32eb..8b0fa6875170 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json @@ -240,7 +240,7 @@ "method": { "type": "string", "const": "CDC", - "order": 0 + "order": 1 }, "plugin": { "type": "string", @@ -248,26 +248,26 @@ "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], "const": "pgoutput", - "order": 1 + "order": 2 }, "replication_slot": { "type": "string", "title": "Replication Slot", "description": "A plugin logical replication slot. Read about replication slots.", - "order": 2 + "order": 3 }, "publication": { "type": "string", "title": "Publication", "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", - "order": 3 + "order": 4 }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", "default": 300, - "order": 4, + "order": 5, "min": 120, "max": 1200 } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java index c47aaa9497fc..109c2c02057f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java @@ -18,7 +18,6 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; @@ -36,7 +35,6 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; import io.airbyte.test.utils.PostgreSQLContainerHelper; -import java.sql.JDBCType; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -268,7 +266,7 @@ public boolean supportsSchemas() { } @Override - public AbstractJdbcSource getJdbcSource() { + public AbstractJdbcSource getJdbcSource() { return new PostgresSource(); } @@ -416,11 +414,6 @@ void incrementalTimestampCheck() throws Exception { getTestMessages().get(2))); } - @Override - protected JdbcSourceOperations getSourceOperations() { - return new PostgresSourceOperations(); - } - @Override protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { final List expectedMessages = new ArrayList<>(); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java index dc27b8a5db33..a93c866ad318 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceStrictEncryptTest.java @@ -25,45 +25,112 @@ public class PostgresSourceStrictEncryptTest { + private final PostgresSourceStrictEncrypt source = new PostgresSourceStrictEncrypt(); + private final PostgreSQLContainer postgreSQLContainerNoSSL = new PostgreSQLContainer<>("postgres:13-alpine"); + private final PostgreSQLContainer postgreSQLContainerWithSSL = + new PostgreSQLContainer<>(DockerImageName.parse("marcosmarxm/postgres-ssl:dev").asCompatibleSubstituteFor("postgres")) + .withCommand("postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key"); + private static final List NON_STRICT_SSL_MODES = List.of("disable", "allow", "prefer"); + private static final String SSL_MODE_REQUIRE = "require"; + private static final SshBastionContainer bastion = new SshBastionContainer(); private static final Network network = Network.newNetwork(); @Test - void testCheckWithSSlModeDisable() throws Exception { + void testSSlModesDisableAllowPreferWithTunnelIfServerDoesNotSupportSSL() throws Exception { + + try (PostgreSQLContainer db = postgreSQLContainerNoSSL.withNetwork(network)) { + bastion.initAndStartBastion(network); + db.start(); + + for (String sslmode : NON_STRICT_SSL_MODES) { + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); + } + + } finally { + bastion.stopAndClose(); + } + } + + @Test + void testSSlModesDisableAllowPreferWithTunnelIfServerSupportSSL() throws Exception { + try (PostgreSQLContainer db = postgreSQLContainerWithSSL.withNetwork(network)) { + + bastion.initAndStartBastion(network); + db.start(); + for (String sslmode : NON_STRICT_SSL_MODES) { + + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); + } + } finally { + bastion.stopAndClose(); + } + } + + @Test + void testSSlModesDisableAllowPreferWithFailedTunnelIfServerSupportSSL() throws Exception { + try (PostgreSQLContainer db = postgreSQLContainerWithSSL) { - try (PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) { bastion.initAndStartBastion(network); db.start(); + for (String sslmode : NON_STRICT_SSL_MODES) { - // stop to enforce ssl for ssl_mode disable - final ImmutableMap.Builder builderWithSSLModeDisable = getDatabaseConfigBuilderWithSSLMode(db, "disable"); - final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModeDisable); - final AirbyteConnectionStatus connectionStatusForDisabledMode = new PostgresSourceStrictEncrypt().check(configWithSSLModeDisable); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForDisabledMode.getStatus()); + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode); + assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); + assertTrue(connectionStatus.getMessage().contains("Connection is not available")); + } } finally { bastion.stopAndClose(); } } @Test - void testCheckWithSSlModePrefer() throws Exception { + void testSSlRequiredWithTunnelIfServerDoesNotSupportSSL() throws Exception { - try (PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network)) { + try (PostgreSQLContainer db = postgreSQLContainerNoSSL.withNetwork(network)) { bastion.initAndStartBastion(network); db.start(); - // continue to enforce ssl because ssl mode is prefer - final ImmutableMap.Builder builderWithSSLModePrefer = getDatabaseConfigBuilderWithSSLMode(db, "prefer"); - final JsonNode configWithSSLModePrefer = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModePrefer); - final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(configWithSSLModePrefer); - assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatusForPreferredMode.getStatus()); - assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatusForPreferredMode.getMessage()); + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, SSL_MODE_REQUIRE); + assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); + assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatus.getMessage()); } finally { bastion.stopAndClose(); } } + @Test + void testSSlRequiredNoTunnelIfServerSupportSSL() throws Exception { + + try (PostgreSQLContainer db = postgreSQLContainerWithSSL) { + db.start(); + + final ImmutableMap configBuilderWithSSLMode = getDatabaseConfigBuilderWithSSLMode(db, SSL_MODE_REQUIRE).build(); + final JsonNode config = Jsons.jsonNode(configBuilderWithSSLMode); + addNoTunnel((ObjectNode) config); + final AirbyteConnectionStatus connectionStatus = source.check(config); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); + } + } + + @Test + void testStrictSSLSecuredWithTunnel() throws Exception { + + try (PostgreSQLContainer db = postgreSQLContainerWithSSL.withNetwork(network)) { + + bastion.initAndStartBastion(network); + db.start(); + + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, SSL_MODE_REQUIRE); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); + } finally { + bastion.stopAndClose(); + } + } + private ImmutableMap.Builder getDatabaseConfigBuilderWithSSLMode(PostgreSQLContainer db, String sslMode) { return ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo() @@ -94,53 +161,26 @@ private JsonNode getMockedSSLConfig(String sslMode) { @Test void testSslModesUnsecuredNoTunnel() throws Exception { - for (String sslMode : List.of("disable", "allow", "prefer")) { + for (String sslMode : NON_STRICT_SSL_MODES) { final JsonNode config = getMockedSSLConfig(sslMode); - ((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build())); + addNoTunnel((ObjectNode) config); - final AirbyteConnectionStatus actual = new PostgresSourceStrictEncrypt().check(config); - assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); - assertTrue(actual.getMessage().contains("Unsecured connection not allowed")); + final AirbyteConnectionStatus connectionStatus = source.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); + assertTrue(connectionStatus.getMessage().contains("Unsecured connection not allowed")); } } - @Test - void testSslModeRequiredNoTunnel() throws Exception { - - try (PostgreSQLContainer db = - new PostgreSQLContainer<>(DockerImageName.parse("marcosmarxm/postgres-ssl:dev").asCompatibleSubstituteFor("postgres")) - .withCommand("postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key")) { - db.start(); - - final ImmutableMap configBuilderWithSslModeRequire = getDatabaseConfigBuilderWithSSLMode(db, "require").build(); - final JsonNode config = Jsons.jsonNode(configBuilderWithSslModeRequire); - ((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build())); - final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(config); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForPreferredMode.getStatus()); - } + private AirbyteConnectionStatus checkWithTunnel(PostgreSQLContainer db, String sslmode) throws Exception { + final ImmutableMap.Builder configBuilderWithSSLMode = getDatabaseConfigBuilderWithSSLMode(db, sslmode); + final JsonNode configWithSSLModeDisable = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, configBuilderWithSSLMode); + return source.check(configWithSSLModeDisable); } - @Test - void testStrictSSLSecuredWithTunnel() throws Exception { - try (PostgreSQLContainer db = - new PostgreSQLContainer<>(DockerImageName.parse("marcosmarxm/postgres-ssl:dev").asCompatibleSubstituteFor("postgres")) - .withCommand("postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key") - .withNetwork(network)) { - - bastion.initAndStartBastion(network); - db.start(); - - final ImmutableMap.Builder builderWithSSLModePrefer = getDatabaseConfigBuilderWithSSLMode(db, "require"); - final JsonNode configWithSslAndSsh = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, builderWithSSLModePrefer); - final AirbyteConnectionStatus connectionStatusForPreferredMode = new PostgresSourceStrictEncrypt().check(configWithSslAndSsh); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatusForPreferredMode.getStatus()); - } finally { - bastion.stopAndClose(); - } + private static void addNoTunnel(ObjectNode config) { + config.putIfAbsent("tunnel_method", Jsons.jsonNode(ImmutableMap.builder() + .put("tunnel_method", "NO_TUNNEL") + .build())); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 99712900e88d..6676e203ce5d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -199,18 +199,6 @@ private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbN .build()); } - private JsonNode getConfigWithSsl(final PostgreSQLContainer psqlDb, final String dbName) { - return Jsons.jsonNode(ImmutableMap.builder() - .put("host", psqlDb.getHost()) - .put("port", psqlDb.getFirstMappedPort()) - .put("database", dbName) - .put("schemas", List.of(SCHEMA_NAME)) - .put("username", psqlDb.getUsername()) - .put("password", psqlDb.getPassword()) - .put("ssl", true) - .build()); - } - private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbName, final String user, final String password) { return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, psqlDb.getHost()) @@ -481,22 +469,6 @@ void testIsCdc() { assertTrue(PostgresUtils.isCdc(config)); } - @Test - void testGetDefaultConnectionPropertiesWithoutSsl() { - final JsonNode config = getConfig(PSQL_DB, dbName); - final Map defaultConnectionProperties = new PostgresSource().getDefaultConnectionProperties(config); - assertEquals(defaultConnectionProperties, Collections.emptyMap()); - }; - - @Test - void testGetDefaultConnectionPropertiesWithSsl() { - final JsonNode config = getConfigWithSsl(PSQL_DB, dbName); - final Map defaultConnectionProperties = new PostgresSource().getDefaultConnectionProperties(config); - assertEquals(defaultConnectionProperties, ImmutableMap.of( - "ssl", "true", - "sslmode", "require")); - }; - @Test void testGetUsername() { final String username = "airbyte-user"; @@ -568,7 +540,8 @@ private JsonNode buildConfigEscapingNeeded() { JdbcUtils.HOST_KEY, "localhost", JdbcUtils.PORT_KEY, 1111, JdbcUtils.USERNAME_KEY, "user", - JdbcUtils.DATABASE_KEY, "db/foo")); + JdbcUtils.DATABASE_KEY, "db/foo", + JdbcUtils.SSL_KEY, "false")); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json index 52865891ce86..596d49c26e02 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test/resources/expected_spec.json @@ -220,7 +220,7 @@ "method": { "type": "string", "const": "CDC", - "order": 0 + "order": 1 }, "plugin": { "type": "string", @@ -228,26 +228,26 @@ "description": "A logical decoding plugin installed on the PostgreSQL server. The `pgoutput` plugin is used by default. If the replication table contains a lot of big jsonb values it is recommended to use `wal2json` plugin. Read more about selecting replication plugins.", "enum": ["pgoutput", "wal2json"], "const": "pgoutput", - "order": 1 + "order": 2 }, "replication_slot": { "type": "string", "title": "Replication Slot", "description": "A plugin logical replication slot. Read about replication slots.", - "order": 2 + "order": 3 }, "publication": { "type": "string", "title": "Publication", "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", - "order": 3 + "order": 4 }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", "default": 300, - "order": 4, + "order": 5, "min": 120, "max": 1200 } diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py index d4d1f9591eb2..ccb39e96eef1 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py @@ -10,7 +10,7 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py index 3031ea896510..09b3bbe717da 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py @@ -10,7 +10,7 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 09366612938d..e2ee118d8ef1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -486,6 +486,13 @@ class TicketComments(SourceZendeskSupportTicketEventsExportStream): sideload_param = "comment_events" event_type = "Comment" + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + for record in super().parse_response(response, **kwargs): + # https://github.com/airbytehq/oncall/issues/1001 + if type(record.get("via")) is not dict: + record["via"] = None + yield record + class Groups(SourceZendeskSupportStream): """Groups stream: https://developer.zendesk.com/api-reference/ticketing/groups/groups/""" diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java index 23ae35754c7f..41abede3e79b 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java @@ -63,6 +63,11 @@ public static final class Tags { */ public static final String DOCKER_IMAGE_KEY = "docker_image"; + /** + * Name of the APM trace tag that holds the failure origin(s) associated with the trace. + */ + public static final String FAILURE_ORIGINS_KEY = "failure_origins"; + /** * Name of the APM trace tag that holds the job ID value associated with the trace. */ diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricTags.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricTags.java index 6378b0c12fcd..99e6fbe60469 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricTags.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricTags.java @@ -29,7 +29,7 @@ public static String getReleaseStage(final ReleaseStage stage) { } public static String getFailureOrigin(final FailureOrigin origin) { - return origin != null ? origin.value() : UNKNOWN; + return origin != null ? origin.value() : FailureOrigin.UNKNOWN.value(); } public static String getJobStatus(final JobStatus status) { diff --git a/airbyte-metrics/reporter/Dockerfile b/airbyte-metrics/reporter/Dockerfile index 3122c74801bc..042c3fde2eaa 100644 --- a/airbyte-metrics/reporter/Dockerfile +++ b/airbyte-metrics/reporter/Dockerfile @@ -1,7 +1,7 @@ ARG JDK_IMAGE=airbyte/airbyte-base-java-image:1.0 FROM ${JDK_IMAGE} AS metrics-reporter -ARG VERSION=0.40.21 +ARG VERSION=0.40.22 ENV APPLICATION airbyte-metrics-reporter ENV VERSION ${VERSION} diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/JsonSchemaType.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/JsonSchemaType.java index 4885bac3fb8c..1cb8ed6c8ad0 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/JsonSchemaType.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/JsonSchemaType.java @@ -23,6 +23,7 @@ public class JsonSchemaType { public static final String CONTENT_ENCODING = "contentEncoding"; public static final String BASE_64 = "base64"; public static final String AIRBYTE_TYPE = "airbyte_type"; + public static final String ITEMS = "items"; public static final JsonSchemaType STRING = JsonSchemaType.builder(JsonSchemaPrimitive.STRING).build(); public static final JsonSchemaType NUMBER = JsonSchemaType.builder(JsonSchemaPrimitive.NUMBER).build(); @@ -48,13 +49,18 @@ public class JsonSchemaType { JsonSchemaType.builder(JsonSchemaPrimitive.STRING) .withFormat(DATE_TIME) .withAirbyteType(TIMESTAMP_WITHOUT_TIMEZONE).build(); - public static final JsonSchemaType STRING_DATE = JsonSchemaType.builder(JsonSchemaPrimitive.STRING) - .withFormat(DATE).build(); - public static final JsonSchemaType NUMBER_BIGINT = JsonSchemaType.builder(JsonSchemaPrimitive.STRING).withAirbyteType("big_integer").build(); + public static final JsonSchemaType STRING_DATE = + JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .withFormat(DATE) + .build(); + public static final JsonSchemaType NUMBER_BIGINT = + JsonSchemaType.builder(JsonSchemaPrimitive.STRING) + .withAirbyteType("big_integer") + .build(); - private final Map jsonSchemaTypeMap; + private final Map jsonSchemaTypeMap; - private JsonSchemaType(final Map jsonSchemaTypeMap) { + private JsonSchemaType(final Map jsonSchemaTypeMap) { this.jsonSchemaTypeMap = jsonSchemaTypeMap; } @@ -62,13 +68,13 @@ public static Builder builder(final JsonSchemaPrimitive type) { return new Builder(type); } - public Map getJsonSchemaTypeMap() { + public Map getJsonSchemaTypeMap() { return jsonSchemaTypeMap; } public static class Builder { - private final ImmutableMap.Builder typeMapBuilder; + private final ImmutableMap.Builder typeMapBuilder; private Builder(final JsonSchemaPrimitive type) { typeMapBuilder = ImmutableMap.builder(); @@ -94,6 +100,11 @@ public JsonSchemaType build() { return new JsonSchemaType(typeMapBuilder.build()); } + public Builder withItems(final JsonSchemaType items) { + typeMapBuilder.put(ITEMS, items.getJsonSchemaTypeMap()); + return this; + } + } @Override diff --git a/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 5c6baf92524f..b51e9d80db78 100644 --- a/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -217,7 +217,11 @@ definitions: type: string type: title: "estimate type" # this title is required to avoid python codegen conflicts with the "type" parameter in AirbyteMessage. See https://github.com/airbytehq/airbyte/pull/12581 - description: The type of estimate + description: > + Estimates are either per-stream (STREAM) or for the entire sync (SYNC). + STREAM is preferred, and requires the source to count how many records are about to be emitted per-stream (e.g. there will be 100 rows from this table emitted). + For the rare source which cannot tell which stream a record belongs to before reading (e.g. CDC databases), SYNC estimates can be emitted. + Sources should not emit both STREAM and SOURCE estimates within a sync. type: string enum: - STREAM diff --git a/airbyte-proxy/Dockerfile b/airbyte-proxy/Dockerfile index 873312fa85c5..7e3d493677ff 100644 --- a/airbyte-proxy/Dockerfile +++ b/airbyte-proxy/Dockerfile @@ -2,16 +2,17 @@ FROM nginx:latest -ARG VERSION=0.40.21 +ARG VERSION=0.40.22 ENV APPLICATION airbyte-proxy ENV VERSION ${VERSION} RUN apt-get update -y && apt-get install -y apache2-utils && rm -rf /var/lib/apt/lists/* -# This variable can be used to update the destintion containers that Nginx proxies to. +# This variable can be used to update the destination containers that Nginx proxies to. ENV PROXY_PASS_WEB "http://airbyte-webapp:80" ENV PROXY_PASS_API "http://airbyte-server:8001" +ENV CONNECTOR_BUILDER_SERVER_API "http://airbyte-connector-builder-server:80" # Nginx config file WORKDIR / diff --git a/airbyte-proxy/nginx-auth.conf.template b/airbyte-proxy/nginx-auth.conf.template index 9dcf9804cd4a..2a6f279329be 100644 --- a/airbyte-proxy/nginx-auth.conf.template +++ b/airbyte-proxy/nginx-auth.conf.template @@ -42,4 +42,25 @@ http { } } } + + server { + listen 8003; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + auth_basic "Welcome to Airbyte"; + auth_basic_user_file /etc/nginx/.htpasswd; + + proxy_pass "${CONNECTOR_BUILDER_SERVER_API}"; + + error_page 401 /etc/nginx/401.html; + location ~ (401.html)$ { + alias /etc/nginx/$1; + auth_basic off; + } + } + } } diff --git a/airbyte-proxy/nginx-no-auth.conf.template b/airbyte-proxy/nginx-no-auth.conf.template index 577dd7ef7f11..ab3a2e8d9e67 100644 --- a/airbyte-proxy/nginx-no-auth.conf.template +++ b/airbyte-proxy/nginx-no-auth.conf.template @@ -24,4 +24,16 @@ http { proxy_pass "${PROXY_PASS_API}"; } } + + server { + listen 8003; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass "${CONNECTOR_BUILDER_SERVER_API}"; + } + } } diff --git a/airbyte-proxy/run.sh b/airbyte-proxy/run.sh index afa840adb70e..032c60e4996c 100644 --- a/airbyte-proxy/run.sh +++ b/airbyte-proxy/run.sh @@ -16,7 +16,7 @@ else TEMPLATE_PATH="/etc/nginx/templates/nginx-auth.conf.template" fi -envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${PROXY_PASS_RESOLVER}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf +envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf echo "starting nginx..." nginx -v diff --git a/airbyte-proxy/test.sh b/airbyte-proxy/test.sh index 6d26885912e7..4abec191f4f8 100755 --- a/airbyte-proxy/test.sh +++ b/airbyte-proxy/test.sh @@ -11,7 +11,7 @@ VERSION="${VERSION:-dev}" # defaults to "dev", otherwise it is set by environmen echo "testing with proxy container airbyte/proxy:$VERSION" function start_container () { - CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --name $NAME airbyte/proxy:$VERSION" + CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION" echo $CMD eval $CMD wait_for_docker; diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index ed461b234c7a..0c03fb7a0103 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -3,7 +3,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 -ARG VERSION=0.40.21 +ARG VERSION=0.40.22 ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ApiHelper.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ApiHelper.java index 5164ea9135f7..b24eda21e754 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ApiHelper.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ApiHelper.java @@ -9,6 +9,7 @@ import io.airbyte.server.errors.IdNotFoundKnownException; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; +import org.slf4j.LoggerFactory; public class ApiHelper { @@ -23,6 +24,9 @@ static T execute(final HandlerCall call) { String.format("The provided configuration does not fulfill the specification. Errors: %s", e.getMessage()), e); } catch (final IOException e) { throw new RuntimeException(e); + } catch (final Exception e) { + LoggerFactory.getLogger(ApiHelper.class).error("Unexpected Exception", e); + throw e; } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java index da5df2f211b4..aa91c5e9b71e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java @@ -133,6 +133,10 @@ public static StandardSync.Status toPersistenceStatus(final ConnectionStatus api return Enums.convertTo(apiStatus, StandardSync.Status.class); } + public static StandardSync.NonBreakingChangesPreference toPersistenceNonBreakingChangesPreference(final NonBreakingChangesPreference preference) { + return Enums.convertTo(preference, StandardSync.NonBreakingChangesPreference.class); + } + public static Geography toApiGeography(final io.airbyte.config.Geography geography) { return Enums.convertTo(geography, Geography.class); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java index 41c013e2a43b..d68380a3ffc1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java @@ -147,7 +147,9 @@ public ConnectionRead createConnection(final ConnectionCreate connectionCreate) .withStatus(ApiPojoConverters.toPersistenceStatus(connectionCreate.getStatus())) .withSourceCatalogId(connectionCreate.getSourceCatalogId()) .withGeography(getGeographyFromConnectionCreateOrWorkspace(connectionCreate)) - .withBreakingChange(false); + .withBreakingChange(false) + .withNonBreakingChangesPreference( + ApiPojoConverters.toPersistenceNonBreakingChangesPreference(connectionCreate.getNonBreakingChangesPreference())); if (connectionCreate.getResourceRequirements() != null) { standardSync.withResourceRequirements(ApiPojoConverters.resourceRequirementsToInternal(connectionCreate.getResourceRequirements())); } @@ -352,6 +354,14 @@ private static void applyPatchToStandardSync(final StandardSync sync, final Conn if (patch.getBreakingChange() != null) { sync.setBreakingChange(patch.getBreakingChange()); } + + if (patch.getNotifySchemaChanges() != null) { + sync.setNotifySchemaChanges(patch.getNotifySchemaChanges()); + } + + if (patch.getNonBreakingChangesPreference() != null) { + sync.setNonBreakingChangesPreference(ApiPojoConverters.toPersistenceNonBreakingChangesPreference(patch.getNonBreakingChangesPreference())); + } } private void validateConnectionPatch(final WorkspaceHelper workspaceHelper, final StandardSync persistedSync, final ConnectionUpdate patch) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index 9931d3665f93..a76d0e6bf61a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -621,6 +621,7 @@ protected static ConnectionCreate toConnectionCreate(final WebBackendConnectionC connectionCreate.resourceRequirements(webBackendConnectionCreate.getResourceRequirements()); connectionCreate.sourceCatalogId(webBackendConnectionCreate.getSourceCatalogId()); connectionCreate.geography(webBackendConnectionCreate.getGeography()); + connectionCreate.nonBreakingChangesPreference(webBackendConnectionCreate.getNonBreakingChangesPreference()); return connectionCreate; } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index c718a9c518c9..78b921135298 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -43,6 +43,7 @@ import io.airbyte.api.model.generated.JobStatus; import io.airbyte.api.model.generated.JobWithAttemptsRead; import io.airbyte.api.model.generated.NamespaceDefinitionType; +import io.airbyte.api.model.generated.NonBreakingChangesPreference; import io.airbyte.api.model.generated.OperationRead; import io.airbyte.api.model.generated.OperationReadList; import io.airbyte.api.model.generated.OperationUpdate; @@ -495,7 +496,8 @@ void testToConnectionCreate() throws IOException { .schedule(schedule) .syncCatalog(catalog) .sourceCatalogId(sourceCatalogId) - .geography(Geography.US); + .geography(Geography.US) + .nonBreakingChangesPreference(NonBreakingChangesPreference.DISABLE); final List operationIds = List.of(newOperationId); @@ -511,7 +513,8 @@ void testToConnectionCreate() throws IOException { .schedule(schedule) .syncCatalog(catalog) .sourceCatalogId(sourceCatalogId) - .geography(Geography.US); + .geography(Geography.US) + .nonBreakingChangesPreference(NonBreakingChangesPreference.DISABLE); final ConnectionCreate actual = WebBackendConnectionsHandler.toConnectionCreate(input, operationIds); @@ -539,7 +542,9 @@ void testToConnectionPatch() throws IOException { .schedule(schedule) .name(standardSync.getName()) .syncCatalog(catalog) - .geography(Geography.US); + .geography(Geography.US) + .nonBreakingChangesPreference(NonBreakingChangesPreference.DISABLE) + .notifySchemaChanges(false); final List operationIds = List.of(newOperationId); @@ -553,7 +558,9 @@ void testToConnectionPatch() throws IOException { .schedule(schedule) .name(standardSync.getName()) .syncCatalog(catalog) - .geography(Geography.US); + .geography(Geography.US) + .nonBreakingChangesPreference(NonBreakingChangesPreference.DISABLE) + .notifySchemaChanges(false); final ConnectionUpdate actual = WebBackendConnectionsHandler.toConnectionPatch(input, operationIds); diff --git a/airbyte-webapp-e2e-tests/cypress.json b/airbyte-webapp-e2e-tests/cypress.json index 41fe0938fea1..9dbca102c121 100644 --- a/airbyte-webapp-e2e-tests/cypress.json +++ b/airbyte-webapp-e2e-tests/cypress.json @@ -8,5 +8,12 @@ "runMode": 2, "openMode": 0 }, - "defaultCommandTimeout": 10000 + "defaultCommandTimeout": 10000, + "db": { + "user": "postgres", + "host": "localhost", + "database": "airbyte_ci", + "password": "secret_password", + "port":5433 + } } diff --git a/airbyte-webapp-e2e-tests/cypress/commands/common.ts b/airbyte-webapp-e2e-tests/cypress/commands/common.ts index 3c3284e2fa79..e5b6aa51c54c 100644 --- a/airbyte-webapp-e2e-tests/cypress/commands/common.ts +++ b/airbyte-webapp-e2e-tests/cypress/commands/common.ts @@ -30,6 +30,6 @@ export const fillEmail = (email: string) => { // useful for ensuring that a name is unique from one test run to the next export const appendRandomString = (string: string) => { - const randomString = Math.random().toString(36).substring(2,10); + const randomString = Math.random().toString(36).substring(2, 10); return string + " _" + randomString; -} +}; diff --git a/airbyte-webapp-e2e-tests/cypress/commands/db/db.ts b/airbyte-webapp-e2e-tests/cypress/commands/db/db.ts new file mode 100644 index 000000000000..fe5524ac60af --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/commands/db/db.ts @@ -0,0 +1,51 @@ +import { + alterCitiesTableQuery, + createCarsTableQuery, + createCitiesTableQuery, + createUsersTableQuery, + dropCarsTableQuery, + dropCitiesTableQuery, + dropUsersTableQuery, + insertCitiesTableQuery, + insertUsersTableQuery, +} from "./queries"; + +/** + * Wrapper for DB Query Cypress task + * @param queryString + */ +export const runDbQuery = (queryString: string) => cy.task("dbQuery", { query: queryString }); + +interface TableExistsResponse { + exists: boolean; +} +/** + * Function for composing the query for checking the existence of a table + * @param tableName + * @return string + */ +const composeIsTableExistQuery = (tableName: string) => + `SELECT EXISTS (SELECT FROM pg_tables + WHERE + schemaname = 'public' AND + tablename = '${tableName}' + )`; + +export const populateDBSource = () => { + runDbQuery(createUsersTableQuery); + runDbQuery(insertUsersTableQuery); + runDbQuery(createCitiesTableQuery); + runDbQuery(insertCitiesTableQuery); +}; + +export const makeChangesInDBSource = () => { + runDbQuery(dropUsersTableQuery); + runDbQuery(alterCitiesTableQuery); + runDbQuery(createCarsTableQuery); +}; + +export const cleanDBSource = () => { + runDbQuery(dropUsersTableQuery); + runDbQuery(dropCitiesTableQuery); + runDbQuery(dropCarsTableQuery); +}; diff --git a/airbyte-webapp-e2e-tests/cypress/commands/db/index.ts b/airbyte-webapp-e2e-tests/cypress/commands/db/index.ts new file mode 100644 index 000000000000..9071a7574a6a --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/commands/db/index.ts @@ -0,0 +1 @@ +export { populateDBSource, makeChangesInDBSource, cleanDBSource } from "./db"; diff --git a/airbyte-webapp-e2e-tests/cypress/commands/db/queries.ts b/airbyte-webapp-e2e-tests/cypress/commands/db/queries.ts new file mode 100644 index 000000000000..2ffc964c3b97 --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/commands/db/queries.ts @@ -0,0 +1,33 @@ +// Users table +export const createUsersTableQuery = ` + CREATE TABLE users(id SERIAL PRIMARY KEY, col1 VARCHAR(200));`; +export const insertUsersTableQuery = ` + INSERT INTO public.users(col1) VALUES('record1'); + INSERT INTO public.users(col1) VALUES('record2'); + INSERT INTO public.users(col1) VALUES('record3');`; + +export const dropUsersTableQuery = ` + DROP TABLE IF EXISTS users;`; + +// Cities table +export const createCitiesTableQuery = ` + CREATE TABLE cities(city_code VARCHAR(8), city VARCHAR(200));`; + +export const insertCitiesTableQuery = ` + INSERT INTO public.cities(city_code, city) VALUES('BCN', 'Barcelona'); + INSERT INTO public.cities(city_code, city) VALUES('MAD', 'Madrid'); + INSERT INTO public.cities(city_code, city) VALUES('VAL', 'Valencia')`; + +export const alterCitiesTableQuery = ` + ALTER TABLE public.cities + DROP COLUMN "city_code", + ADD COLUMN "state" text, + ADD COLUMN "country" text;`; +export const dropCitiesTableQuery = ` + DROP TABLE IF EXISTS cities;`; + +// Cars table +export const createCarsTableQuery = ` + CREATE TABLE cars(id SERIAL PRIMARY KEY, mark VARCHAR(200), model VARCHAR(200), color VARCHAR(200));`; +export const dropCarsTableQuery = ` + DROP TABLE IF EXISTS cars;`; diff --git a/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.ts b/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.ts index 6750d7b18627..655ab5cd2411 100644 --- a/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.ts +++ b/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.ts @@ -11,10 +11,22 @@ import { setupDestinationNamespaceCustomFormat, selectFullAppendSyncMode, checkSuccessResult, + refreshSourceSchemaBtnClick, + resetModalSaveBtnClick, + toggleStreamEnabledState, } from "pages/replicationPage"; import { openSourceDestinationFromGrid, goToSourcePage } from "pages/sourcePage"; import { goToSettingsPage } from "pages/settingsConnectionPage"; -import { update } from "cypress/types/lodash"; +import { cleanDBSource, makeChangesInDBSource, populateDBSource } from "../commands/db"; +import { + catalogDiffModal, + newFieldsTable, + newStreamsTable, + removedFieldsTable, + removedStreamsTable, + toggleStreamWithChangesAccordion, +} from "../pages/modals/catalogDiffModal"; +import { updateSchemaModalConfirmBtnClick } from "../pages/modals/updateSchemaModal"; describe("Connection main actions", () => { beforeEach(() => { @@ -127,7 +139,7 @@ describe("Connection main actions", () => { deleteDestination(destName); }); - it("creates a connection, then edits the schedule type", () => { + it("Creates a connection, then edits the schedule type", () => { const sourceName = appendRandomString("Test connection source cypress PokeAPI"); const destName = appendRandomString("Test connection destination cypress"); @@ -171,7 +183,11 @@ describe("Connection main actions", () => { let loadedConnection: any = null; // Should be a WebBackendConnectionRead cy.wait("@getConnection").then((interception) => { - const { scheduleType: readScheduleType, scheduleData: readScheduleData, ...connectionRead } = interception.response?.body; + const { + scheduleType: readScheduleType, + scheduleData: readScheduleData, + ...connectionRead + } = interception.response?.body; loadedConnection = connectionRead; expect(loadedConnection).not.to.eq(null); @@ -201,6 +217,54 @@ describe("Connection main actions", () => { deleteDestination(destName); }); + it("Create a connection, update data in source, show diff modal, reset streams", () => { + cy.intercept("/api/v1/web_backend/connections/update").as("updateConnection"); + + populateDBSource(); + const sourceName = appendRandomString( + "Test refresh source schema with changed data - connection Postgres source cypress" + ); + const destName = appendRandomString( + "Test refresh source schema with changed data - connection Local JSON destination cypress" + ); + + createTestConnection(sourceName, destName); + cy.get("div").contains(sourceName).should("exist"); + cy.get("div").contains(destName).should("exist"); + + makeChangesInDBSource(); + openSourceDestinationFromGrid(sourceName); + goToReplicationTab(); + refreshSourceSchemaBtnClick(); + + cy.get(catalogDiffModal).should("exist"); + + cy.get(removedStreamsTable).should("contain", "users"); + + cy.get(newStreamsTable).should("contain", "cars"); + + toggleStreamWithChangesAccordion("cities"); + cy.get(removedFieldsTable).should("contain", "city_code"); + cy.get(newFieldsTable).children().should("contain", "country").and("contain", "state"); + + updateSchemaModalConfirmBtnClick(); + + toggleStreamEnabledState("cars"); + + submitButtonClick(); + resetModalSaveBtnClick(); + + cy.wait("@updateConnection").then((interception) => { + assert.isNotNull(interception.response?.statusCode, "200"); + }); + + checkSuccessResult(); + + deleteSource(sourceName); + deleteDestination(destName); + cleanDBSource(); + }); + it("Delete connection", () => { const sourceName = "Test delete connection source cypress"; const destName = "Test delete connection destination cypress"; diff --git a/airbyte-webapp-e2e-tests/cypress/pages/modals/catalogDiffModal.ts b/airbyte-webapp-e2e-tests/cypress/pages/modals/catalogDiffModal.ts new file mode 100644 index 000000000000..e625f0ee3c51 --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/pages/modals/catalogDiffModal.ts @@ -0,0 +1,11 @@ +export const catalogDiffModal = "[data-testid='catalog-diff-modal']"; +export const removedStreamsTable = "table[aria-label='removed streams table']"; +export const newStreamsTable = "table[aria-label='new streams table']"; +const streamWithChangesToggleBtn = (streamName: string) => + `button[data-testid='toggle-accordion-${streamName}-stream']`; +export const removedFieldsTable = "table[aria-label='removed fields']"; +export const newFieldsTable = "table[aria-label='new fields']"; + +export const toggleStreamWithChangesAccordion = (streamName: string) => { + cy.get(streamWithChangesToggleBtn(streamName)).click(); +}; diff --git a/airbyte-webapp-e2e-tests/cypress/pages/modals/updateSchemaModal.ts b/airbyte-webapp-e2e-tests/cypress/pages/modals/updateSchemaModal.ts new file mode 100644 index 000000000000..07186ac5aaf9 --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/pages/modals/updateSchemaModal.ts @@ -0,0 +1,3 @@ +export const updateSchemaModalConfirmBtnClick = () => { + cy.get("[data-testid='update-schema-confirm-btn']").click(); +}; \ No newline at end of file diff --git a/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts b/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts index ec9fb8f93635..d12aa36514f4 100644 --- a/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts +++ b/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts @@ -10,6 +10,8 @@ const syncModeDropdown = "div[data-testid='syncSettingsDropdown'] input"; const successResult = "div[data-id='success-result']"; const saveStreamChangesButton = "button[data-testid='resetModal-save']"; const connectionNameInput = "input[data-testid='connectionName']"; +const refreshSourceSchemaButton = "button[data-testid='refresh-source-schema-btn']"; +const streamSyncEnabledSwitch = (streamName: string) => `[data-testid='${streamName}-stream-sync-switch']`; export const goToReplicationTab = () => { cy.get(replicationTab).click(); @@ -42,6 +44,16 @@ export const setupDestinationNamespaceSourceFormat = () => { cy.get(destinationNamespaceSource).click(); }; +export const refreshSourceSchemaBtnClick = () => { + cy.get(refreshSourceSchemaButton).click(); +}; + + + +export const resetModalSaveBtnClick = () => { + cy.get("[data-testid='resetModal-save']").click(); +}; + export const selectFullAppendSyncMode = () => { cy.get(syncModeDropdown).first().click({ force: true }); @@ -57,3 +69,7 @@ export const checkSuccessResult = () => { export const confirmStreamConfigurationChangedPopup = () => { cy.get(saveStreamChangesButton).click(); }; + +export const toggleStreamEnabledState = (streamName: string) => { + cy.get(streamSyncEnabledSwitch(streamName)).check({ force: true }); +}; diff --git a/airbyte-webapp-e2e-tests/cypress/plugins/index.ts b/airbyte-webapp-e2e-tests/cypress/plugins/index.ts index 8dd144a6c1a9..e546c1e5f1e5 100644 --- a/airbyte-webapp-e2e-tests/cypress/plugins/index.ts +++ b/airbyte-webapp-e2e-tests/cypress/plugins/index.ts @@ -12,10 +12,16 @@ // This function is called when a project is opened or re-opened (e.g. due to // the project's config changing) +import Cypress from "cypress" + + /** * @type {Cypress.PluginConfig} */ -module.exports = (on, config) => { +module.exports = (on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions) => { // `on` is used to hook into various events Cypress emits // `config` is the resolved Cypress config + on("task", { + dbQuery:(query)=> require("cypress-postgres")(query.query,query.connection) + }); }; diff --git a/airbyte-webapp-e2e-tests/package-lock.json b/airbyte-webapp-e2e-tests/package-lock.json index 7c7e83c4eea7..7923972b12c3 100644 --- a/airbyte-webapp-e2e-tests/package-lock.json +++ b/airbyte-webapp-e2e-tests/package-lock.json @@ -8,7 +8,9 @@ "name": "airbyte-webapp-e2e-tests", "version": "0.0.0", "devDependencies": { + "@types/node": "^18.11.9", "cypress": "^9.2.0", + "cypress-postgres": "^1.1.1", "eslint-plugin-cypress": "^2.12.1", "prettier": "^2.6.2", "typescript": "^4.5.4" @@ -135,9 +137,9 @@ "peer": true }, "node_modules/@types/node": { - "version": "14.18.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.2.tgz", - "integrity": "sha512-fqtSN5xn/bBzDxMT77C1rJg6CsH/R49E7qsGuvdPJa20HtV5zSTuLJPNfnlyVH3wauKnkHdLggTVkOW/xP9oQg==", + "version": "18.11.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", + "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==", "dev": true }, "node_modules/@types/sinonjs__fake-timers": { @@ -299,6 +301,15 @@ "safer-buffer": "~2.1.0" } }, + "node_modules/assert-options": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/assert-options/-/assert-options-0.6.2.tgz", + "integrity": "sha512-KP9S549XptFAPGYmLRnIjQBL4/Ry8Jx5YNLQZ/l+eejqbTidBMnw4uZSAsUrzBq/lgyqDYqxcTF7cOxZb9gyEw==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", @@ -399,6 +410,15 @@ "node": "*" } }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/cachedir": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.3.0.tgz", @@ -654,6 +674,21 @@ "node": ">=12.0.0" } }, + "node_modules/cypress-postgres": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cypress-postgres/-/cypress-postgres-1.1.1.tgz", + "integrity": "sha512-ewhT84MhUU2FYzREUF2dYMJeIFarI2/8GkoD5xKyT/e7YIdYhJlOaFZSNZRY6OnQAnMWMy0g2p2n6+03HY/HZA==", + "dev": true, + "dependencies": { + "pg-promise": "10.5.8" + } + }, + "node_modules/cypress/node_modules/@types/node": { + "version": "14.18.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.33.tgz", + "integrity": "sha512-qelS/Ra6sacc4loe/3MSjXNL1dNQ/GjxNHVzuChwMfmk7HuycRLVQN2qNY3XahK+fZc5E2szqQSKUyAF0E+2bg==", + "dev": true + }, "node_modules/cypress/node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -1876,6 +1911,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==", + "dev": true + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -1919,6 +1960,113 @@ "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", "dev": true }, + "node_modules/pg": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.2.2.tgz", + "integrity": "sha512-Uni50U0W2CNPM68+zfC/1WWjSO3q/uBSF/Nl7D+1npZGsPSM4/EZt0xSMW2jox1Bn0EfDlnTWnTsM/TrSOtBEA==", + "dev": true, + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.2.3", + "pg-pool": "^3.2.1", + "pg-protocol": "^1.2.5", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==", + "dev": true + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-minify": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/pg-minify/-/pg-minify-1.6.1.tgz", + "integrity": "sha512-ujanxJJB9CSDUvlAOshtjdKAywOPR2vY0a7D+vvgk5rbrYcthZA7TjpN+Z+UwZsz/G/bUexYDT6huE33vYVN0g==", + "dev": true, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/pg-pool": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz", + "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==", + "dev": true, + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-promise": { + "version": "10.5.8", + "resolved": "https://registry.npmjs.org/pg-promise/-/pg-promise-10.5.8.tgz", + "integrity": "sha512-EdLyPd/XlmNsfA2uRKHuCnyLhk5DHPdKGPZmjzpcKfdx6dDZB+nEfSuaNSjReRrM7BmPaV/hSGppt9kG/W2Umw==", + "dev": true, + "dependencies": { + "assert-options": "0.6.2", + "pg": "8.2.2", + "pg-minify": "1.6.1", + "spex": "3.0.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz", + "integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==", + "dev": true + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dev": true, + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pg/node_modules/semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha512-VyFUffiBx8hABJ9HYSTXLRwyZtdDHMzMtFmID1aiNAD2BZppBmJm0Hqw3p2jkgxP9BNt1pQ9RnC49P0EcXf6cA==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dev": true, + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", @@ -1928,6 +2076,45 @@ "node": ">=0.10.0" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dev": true, + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -2183,6 +2370,24 @@ "node": ">=8" } }, + "node_modules/spex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/spex/-/spex-3.0.2.tgz", + "integrity": "sha512-ZNCrOso+oNv5P01HCO4wuxV9Og5rS6ms7gGAqugfBPjx1QwfNXJI3T02ldfaap1O0dlT1sB0Rk+mhDqxt3Z27w==", + "dev": true, + "engines": { + "node": ">=4.5" + } + }, + "node_modules/split2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==", + "dev": true, + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -2496,6 +2701,15 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "engines": { + "node": ">=0.4" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -2619,9 +2833,9 @@ "peer": true }, "@types/node": { - "version": "14.18.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.2.tgz", - "integrity": "sha512-fqtSN5xn/bBzDxMT77C1rJg6CsH/R49E7qsGuvdPJa20HtV5zSTuLJPNfnlyVH3wauKnkHdLggTVkOW/xP9oQg==", + "version": "18.11.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", + "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==", "dev": true }, "@types/sinonjs__fake-timers": { @@ -2736,6 +2950,12 @@ "safer-buffer": "~2.1.0" } }, + "assert-options": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/assert-options/-/assert-options-0.6.2.tgz", + "integrity": "sha512-KP9S549XptFAPGYmLRnIjQBL4/Ry8Jx5YNLQZ/l+eejqbTidBMnw4uZSAsUrzBq/lgyqDYqxcTF7cOxZb9gyEw==", + "dev": true + }, "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", @@ -2821,6 +3041,12 @@ "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=", "dev": true }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "dev": true + }, "cachedir": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.3.0.tgz", @@ -3019,6 +3245,12 @@ "yauzl": "^2.10.0" }, "dependencies": { + "@types/node": { + "version": "14.18.33", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.33.tgz", + "integrity": "sha512-qelS/Ra6sacc4loe/3MSjXNL1dNQ/GjxNHVzuChwMfmk7HuycRLVQN2qNY3XahK+fZc5E2szqQSKUyAF0E+2bg==", + "dev": true + }, "supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -3030,6 +3262,15 @@ } } }, + "cypress-postgres": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cypress-postgres/-/cypress-postgres-1.1.1.tgz", + "integrity": "sha512-ewhT84MhUU2FYzREUF2dYMJeIFarI2/8GkoD5xKyT/e7YIdYhJlOaFZSNZRY6OnQAnMWMy0g2p2n6+03HY/HZA==", + "dev": true, + "requires": { + "pg-promise": "10.5.8" + } + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -3966,6 +4207,12 @@ "aggregate-error": "^3.0.0" } }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==", + "dev": true + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -4000,12 +4247,128 @@ "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", "dev": true }, + "pg": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.2.2.tgz", + "integrity": "sha512-Uni50U0W2CNPM68+zfC/1WWjSO3q/uBSF/Nl7D+1npZGsPSM4/EZt0xSMW2jox1Bn0EfDlnTWnTsM/TrSOtBEA==", + "dev": true, + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.2.3", + "pg-pool": "^3.2.1", + "pg-protocol": "^1.2.5", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + }, + "dependencies": { + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha512-VyFUffiBx8hABJ9HYSTXLRwyZtdDHMzMtFmID1aiNAD2BZppBmJm0Hqw3p2jkgxP9BNt1pQ9RnC49P0EcXf6cA==", + "dev": true + } + } + }, + "pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==", + "dev": true + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "dev": true + }, + "pg-minify": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/pg-minify/-/pg-minify-1.6.1.tgz", + "integrity": "sha512-ujanxJJB9CSDUvlAOshtjdKAywOPR2vY0a7D+vvgk5rbrYcthZA7TjpN+Z+UwZsz/G/bUexYDT6huE33vYVN0g==", + "dev": true + }, + "pg-pool": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz", + "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==", + "dev": true, + "requires": {} + }, + "pg-promise": { + "version": "10.5.8", + "resolved": "https://registry.npmjs.org/pg-promise/-/pg-promise-10.5.8.tgz", + "integrity": "sha512-EdLyPd/XlmNsfA2uRKHuCnyLhk5DHPdKGPZmjzpcKfdx6dDZB+nEfSuaNSjReRrM7BmPaV/hSGppt9kG/W2Umw==", + "dev": true, + "requires": { + "assert-options": "0.6.2", + "pg": "8.2.2", + "pg-minify": "1.6.1", + "spex": "3.0.2" + } + }, + "pg-protocol": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz", + "integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==", + "dev": true + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dev": true, + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dev": true, + "requires": { + "split2": "^4.1.0" + } + }, "pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "dev": true + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "dev": true + }, + "postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "dev": true + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dev": true, + "requires": { + "xtend": "^4.0.0" + } + }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -4183,6 +4546,18 @@ "is-fullwidth-code-point": "^3.0.0" } }, + "spex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/spex/-/spex-3.0.2.tgz", + "integrity": "sha512-ZNCrOso+oNv5P01HCO4wuxV9Og5rS6ms7gGAqugfBPjx1QwfNXJI3T02ldfaap1O0dlT1sB0Rk+mhDqxt3Z27w==", + "dev": true + }, + "split2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==", + "dev": true + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -4420,6 +4795,12 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/airbyte-webapp-e2e-tests/package.json b/airbyte-webapp-e2e-tests/package.json index d9546c2dd18a..ddf6943c0028 100644 --- a/airbyte-webapp-e2e-tests/package.json +++ b/airbyte-webapp-e2e-tests/package.json @@ -21,7 +21,9 @@ ] }, "devDependencies": { + "@types/node": "^18.11.9", "cypress": "^9.2.0", + "cypress-postgres": "^1.1.1", "eslint-plugin-cypress": "^2.12.1", "prettier": "^2.6.2", "typescript": "^4.5.4" diff --git a/airbyte-webapp-e2e-tests/tsconfig.json b/airbyte-webapp-e2e-tests/tsconfig.json index 1bfca79abd6c..5bd50d734540 100644 --- a/airbyte-webapp-e2e-tests/tsconfig.json +++ b/airbyte-webapp-e2e-tests/tsconfig.json @@ -14,7 +14,7 @@ "resolveJsonModule": true, "jsx": "react-jsx", "noFallthroughCasesInSwitch": true, - "types": ["cypress"], + "types": ["cypress", "node"], "lib": ["es2015", "dom"], "isolatedModules": false, "allowJs": true, diff --git a/airbyte-webapp/.env b/airbyte-webapp/.env index b6cf64f3beaf..4e35f5bc0deb 100644 --- a/airbyte-webapp/.env +++ b/airbyte-webapp/.env @@ -3,4 +3,3 @@ REACT_APP_FULL_STORY_ORG=13AXQ4 REACT_APP_SENTRY_DSN= REACT_APP_INTERCOM_APP_ID=nj1oam7s REACT_APP_OSANO=16A0CTTE7vE8m1Qif/67beec9b-e563-4736-bdb4-4fe4adc39d48 -REACT_APP_CONNECTOR_BUILDER_API=/connector-builder-api/ diff --git a/airbyte-webapp/nginx/default.conf.template b/airbyte-webapp/nginx/default.conf.template index 25de5efe4ad3..ee2c2271faf1 100644 --- a/airbyte-webapp/nginx/default.conf.template +++ b/airbyte-webapp/nginx/default.conf.template @@ -25,6 +25,7 @@ server { window.TRACKING_STRATEGY = "$TRACKING_STRATEGY"; window.AIRBYTE_VERSION = "$AIRBYTE_VERSION"; window.API_URL = "$API_URL"; + window.CONNECTOR_BUILDER_API_URL = "$CONNECTOR_BUILDER_API_URL"; '; sub_filter_once on; } diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 5302dcf85c32..aa2b75575128 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "airbyte-webapp", - "version": "0.40.21", + "version": "0.40.22", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "airbyte-webapp", - "version": "0.40.21", + "version": "0.40.22", "dependencies": { "@datadog/browser-rum": "^4.21.2", "@floating-ui/react-dom": "^1.0.0", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index a57995d3aac1..2371fde0d81b 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.40.21", + "version": "0.40.22", "private": true, "engines": { "node": ">=16.0.0" diff --git a/airbyte-webapp/src/components/CreateConnection/__snapshots__/CreateConnectionForm.test.tsx.snap b/airbyte-webapp/src/components/CreateConnection/__snapshots__/CreateConnectionForm.test.tsx.snap index 6d04f054f3c7..b46ae78f9d69 100644 --- a/airbyte-webapp/src/components/CreateConnection/__snapshots__/CreateConnectionForm.test.tsx.snap +++ b/airbyte-webapp/src/components/CreateConnection/__snapshots__/CreateConnectionForm.test.tsx.snap @@ -369,6 +369,7 @@ exports[`CreateConnectionForm should render 1`] = ` diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx index 4f7b5a50568c..52cd6efc35ab 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx @@ -23,7 +23,13 @@ export const DiffAccordion: React.FC = ({ streamTransform }) {({ open }) => ( <> - + { title: formatMessage({ id: "connection.updateSchema.completed" }), preventCancel: true, size: "md", + testId: "catalog-diff-modal", content: ({ onClose }) => ( ), diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionFormFields.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionFormFields.tsx index 93ec5148a4ba..58e0b7c1aa74 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionFormFields.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionFormFields.tsx @@ -124,7 +124,13 @@ export const ConnectionFormFields: React.FC = ({ valu component={SyncCatalogField} isSubmitting={isSubmitting} additionalControl={ - diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 32a7c6ea89ba..0f95ad9d8169 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -10,7 +10,7 @@ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/s && chmod +x kubectl && mv kubectl /usr/local/bin/ # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.40.21 +ARG VERSION=0.40.22 ENV APPLICATION airbyte-workers ENV VERSION ${VERSION} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/ApplicationInitializer.java b/airbyte-workers/src/main/java/io/airbyte/workers/ApplicationInitializer.java index 685f9f2dd5a9..0fcd29327910 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/ApplicationInitializer.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/ApplicationInitializer.java @@ -115,7 +115,7 @@ public class ApplicationInitializer implements ApplicationEventListener trackingStrategy, final DeploymentMode deploymentMode, - Optional jobPersistence, - WorkerEnvironment workerEnvironment, - @Value("${airbyte.role}") String airbyteRole, - AirbyteVersion airbyteVersion, - Optional configRepository) + final Optional jobPersistence, + final WorkerEnvironment workerEnvironment, + @Value("${airbyte.role}") final String airbyteRole, + final AirbyteVersion airbyteVersion, + final Optional configRepository) throws IOException { TrackingClientSingleton.initialize( @@ -63,6 +63,12 @@ public TrackingClient trackingClient(final Optional trackingSt return TrackingClientSingleton.get(); } + @Singleton + @Requires(env = WorkerMode.CONTROL_PLANE) + public OAuthConfigSupplier oAuthConfigSupplier(final ConfigRepository configRepository, final TrackingClient trackingClient) { + return new OAuthConfigSupplier(configRepository, trackingClient); + } + @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) public SyncJobFactory jobFactory( @@ -71,12 +77,12 @@ public SyncJobFactory jobFactory( @Property(name = "airbyte.connector.specific-resource-defaults-enabled", defaultValue = "false") final boolean connectorSpecificResourceDefaultsEnabled, final DefaultJobCreator jobCreator, - final TrackingClient trackingClient) { + final OAuthConfigSupplier oAuthConfigSupplier) { return new DefaultSyncJobFactory( connectorSpecificResourceDefaultsEnabled, jobCreator, configRepository, - new OAuthConfigSupplier(configRepository, trackingClient), + oAuthConfigSupplier, new WorkspaceHelper(configRepository, jobPersistence)); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java index 27531fce2919..78b74764a803 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java @@ -139,7 +139,10 @@ public OUTPUT get() { } LOGGER.info("Executing worker wrapper. Airbyte version: {}", airbyteVersion); - saveWorkflowIdForCancellation(airbyteApiClient); + AirbyteApiClient.retryWithJitter(() -> { + saveWorkflowIdForCancellation(airbyteApiClient); + return null; + }, "save workflow id for cancellation"); final Worker worker = workerSupplier.get(); final CompletableFuture outputFuture = new CompletableFuture<>(); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java index acadfb2bc98a..456af1507218 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java @@ -8,9 +8,11 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.FAILURE_ORIGINS_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; import static io.airbyte.persistence.job.models.AttemptStatus.FAILED; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import datadog.trace.api.Trace; import io.airbyte.commons.docker.DockerUtils; @@ -22,6 +24,7 @@ import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.DestinationConnection; import io.airbyte.config.FailureReason; +import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.JobConfig; import io.airbyte.config.JobOutput; import io.airbyte.config.JobSyncConfig; @@ -44,6 +47,7 @@ import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.errorreporter.JobErrorReporter; import io.airbyte.persistence.job.errorreporter.SyncJobReportingContext; +import io.airbyte.persistence.job.factory.OAuthConfigSupplier; import io.airbyte.persistence.job.factory.SyncJobFactory; import io.airbyte.persistence.job.models.Attempt; import io.airbyte.persistence.job.models.Job; @@ -56,6 +60,7 @@ import io.airbyte.workers.run.TemporalWorkerRunFactory; import io.airbyte.workers.run.WorkerRun; import io.micronaut.context.annotation.Requires; +import io.micronaut.core.util.CollectionUtils; import jakarta.inject.Singleton; import java.io.IOException; import java.nio.file.Path; @@ -67,6 +72,7 @@ import java.util.OptionalLong; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -85,6 +91,7 @@ public class JobCreationAndStatusUpdateActivityImpl implements JobCreationAndSta private final JobCreator jobCreator; private final StreamResetPersistence streamResetPersistence; private final JobErrorReporter jobErrorReporter; + private final OAuthConfigSupplier oAuthConfigSupplier; public JobCreationAndStatusUpdateActivityImpl(final SyncJobFactory jobFactory, final JobPersistence jobPersistence, @@ -96,7 +103,8 @@ public JobCreationAndStatusUpdateActivityImpl(final SyncJobFactory jobFactory, final ConfigRepository configRepository, final JobCreator jobCreator, final StreamResetPersistence streamResetPersistence, - final JobErrorReporter jobErrorReporter) { + final JobErrorReporter jobErrorReporter, + final OAuthConfigSupplier oAuthConfigSupplier) { this.jobFactory = jobFactory; this.jobPersistence = jobPersistence; this.temporalWorkerRunFactory = temporalWorkerRunFactory; @@ -108,6 +116,7 @@ public JobCreationAndStatusUpdateActivityImpl(final SyncJobFactory jobFactory, this.jobCreator = jobCreator; this.streamResetPersistence = streamResetPersistence; this.jobErrorReporter = jobErrorReporter; + this.oAuthConfigSupplier = oAuthConfigSupplier; } @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) @@ -129,6 +138,12 @@ public JobCreationOutput createNewJob(final JobCreationInput input) { if (!streamsToReset.isEmpty()) { final DestinationConnection destination = configRepository.getDestinationConnection(standardSync.getDestinationId()); + final JsonNode destinationConfiguration = oAuthConfigSupplier.injectDestinationOAuthParameters( + destination.getDestinationDefinitionId(), + destination.getWorkspaceId(), + destination.getConfiguration()); + destination.setConfiguration(destinationConfiguration); + final StandardDestinationDefinition destinationDef = configRepository.getStandardDestinationDefinition(destination.getDestinationDefinitionId()); final String destinationImageName = DockerUtils.getTaggedImageName(destinationDef.getDockerRepository(), destinationDef.getDockerImageTag()); @@ -179,9 +194,8 @@ private void emitSrcIdDstIdToReleaseStagesMetric(final UUID srcId, final UUID ds @Override public AttemptCreationOutput createNewAttempt(final AttemptCreationInput input) throws RetryableException { try { - ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, input.getJobId())); - final long jobId = input.getJobId(); + ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, jobId)); final Job createdJob = jobPersistence.getJob(jobId); final WorkerRun workerRun = temporalWorkerRunFactory.create(createdJob); @@ -200,9 +214,8 @@ public AttemptCreationOutput createNewAttempt(final AttemptCreationInput input) @Override public AttemptNumberCreationOutput createNewAttemptNumber(final AttemptCreationInput input) throws RetryableException { try { - ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, input.getJobId())); - final long jobId = input.getJobId(); + ApmTraceUtils.addTagsToTrace(Map.of(JOB_ID_KEY, jobId)); final Job createdJob = jobPersistence.getJob(jobId); final WorkerRun workerRun = temporalWorkerRunFactory.create(createdJob); @@ -221,10 +234,9 @@ public AttemptNumberCreationOutput createNewAttemptNumber(final AttemptCreationI @Override public void jobSuccess(final JobSuccessInput input) { try { - ApmTraceUtils.addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, input.getAttemptId(), JOB_ID_KEY, input.getJobId())); - final long jobId = input.getJobId(); final int attemptId = input.getAttemptId(); + ApmTraceUtils.addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, attemptId, JOB_ID_KEY, jobId)); if (input.getStandardSyncOutput() != null) { final JobOutput jobOutput = new JobOutput().withSync(input.getStandardSyncOutput()); @@ -287,12 +299,13 @@ public void jobFailure(final JobFailureInput input) { @Override public void attemptFailure(final AttemptFailureInput input) { try { - ApmTraceUtils.addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, input.getAttemptId(), JOB_ID_KEY, input.getJobId())); - final int attemptId = input.getAttemptId(); final long jobId = input.getJobId(); final AttemptFailureSummary failureSummary = input.getAttemptFailureSummary(); + ApmTraceUtils.addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, attemptId, JOB_ID_KEY, jobId)); + traceFailures(failureSummary); + jobPersistence.failAttempt(jobId, attemptId); jobPersistence.writeAttemptFailureSummary(jobId, attemptId, failureSummary); @@ -302,11 +315,7 @@ public void attemptFailure(final AttemptFailureInput input) { } emitJobIdToReleaseStagesMetric(OssMetricsRegistry.ATTEMPT_FAILED_BY_RELEASE_STAGE, jobId); - for (final FailureReason reason : failureSummary.getFailures()) { - MetricClientFactory.getMetricClient().count(OssMetricsRegistry.ATTEMPT_FAILED_BY_FAILURE_ORIGIN, 1, - new MetricAttribute(MetricTags.FAILURE_ORIGIN, MetricTags.getFailureOrigin(reason.getFailureOrigin()))); - } - + trackFailures(failureSummary); } catch (final IOException e) { throw new RetryableException(e); } @@ -329,10 +338,9 @@ public void attemptFailureWithAttemptNumber(final AttemptNumberFailureInput inpu @Override public void jobCancelled(final JobCancelledInput input) { try { - ApmTraceUtils.addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, input.getAttemptId(), JOB_ID_KEY, input.getJobId())); - final long jobId = input.getJobId(); final int attemptId = input.getAttemptId(); + ApmTraceUtils.addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, attemptId, JOB_ID_KEY, jobId)); jobPersistence.failAttempt(jobId, attemptId); jobPersistence.writeAttemptFailureSummary(jobId, attemptId, input.getAttemptFailureSummary()); jobPersistence.cancelJob(jobId); @@ -487,4 +495,37 @@ private void trackCompletionForInternalFailure(final Long jobId, jobTracker.trackSyncForInternalFailure(jobId, connectionId, attemptId, Enums.convertTo(status, JobState.class), e); } + /** + * Adds the failure origins to the APM trace. + * + * @param failureSummary The {@link AttemptFailureSummary} containing the failure reason(s). + */ + private void traceFailures(final AttemptFailureSummary failureSummary) { + if (failureSummary != null) { + if (CollectionUtils.isNotEmpty(failureSummary.getFailures())) { + ApmTraceUtils.addTagsToTrace(Map.of(FAILURE_ORIGINS_KEY, failureSummary.getFailures().stream().map(FailureReason::getFailureOrigin).map( + FailureOrigin::name).collect(Collectors.joining(",")))); + } + } else { + ApmTraceUtils.addTagsToTrace(Map.of(FAILURE_ORIGINS_KEY, FailureOrigin.UNKNOWN.value())); + } + } + + /** + * Records a metric for each failure reason. + * + * @param failureSummary The {@link AttemptFailureSummary} containing the failure reason(s). + */ + private void trackFailures(final AttemptFailureSummary failureSummary) { + if (failureSummary != null) { + for (final FailureReason reason : failureSummary.getFailures()) { + MetricClientFactory.getMetricClient().count(OssMetricsRegistry.ATTEMPT_FAILED_BY_FAILURE_ORIGIN, 1, + new MetricAttribute(MetricTags.FAILURE_ORIGIN, MetricTags.getFailureOrigin(reason.getFailureOrigin()))); + } + } else { + MetricClientFactory.getMetricClient().count(OssMetricsRegistry.ATTEMPT_FAILED_BY_FAILURE_ORIGIN, 1, + new MetricAttribute(MetricTags.FAILURE_ORIGIN, FailureOrigin.UNKNOWN.value())); + } + } + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index 768f397ed86f..381d6ebbebfb 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -11,7 +11,6 @@ import datadog.trace.api.Trace; import io.airbyte.api.client.AirbyteApiClient; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.JobIdRequestBody; import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.commons.json.Jsons; @@ -153,11 +152,13 @@ private CheckedSupplier, Exception> getContainerL final WorkerConfigs workerConfigs, final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig, - final Supplier activityContext) - throws ApiException { + final Supplier activityContext) { final JobIdRequestBody id = new JobIdRequestBody(); id.setId(Long.valueOf(jobRunConfig.getJobId())); - final var jobScope = airbyteApiClient.getJobsApi().getJobInfo(id).getJob().getConfigId(); + + final var jobScope = AirbyteApiClient.retryWithJitter( + () -> airbyteApiClient.getJobsApi().getJobInfo(id).getJob().getConfigId(), + "get job scope"); final var connectionId = UUID.fromString(jobScope); return () -> new DbtLauncherWorker( diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index b37e41e8aa33..81410f668bf9 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -11,7 +11,6 @@ import datadog.trace.api.Trace; import io.airbyte.api.client.AirbyteApiClient; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.JobIdRequestBody; import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.commons.json.Jsons; @@ -162,11 +161,12 @@ private CheckedSupplier, Except final WorkerConfigs workerConfigs, final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig, - final Supplier activityContext) - throws ApiException { + final Supplier activityContext) { final JobIdRequestBody id = new JobIdRequestBody(); id.setId(Long.valueOf(jobRunConfig.getJobId())); - final var jobScope = airbyteApiClient.getJobsApi().getJobInfo(id).getJob().getConfigId(); + final var jobScope = AirbyteApiClient.retryWithJitter( + () -> airbyteApiClient.getJobsApi().getJobInfo(id).getJob().getConfigId(), + "get job scope"); final var connectionId = UUID.fromString(jobScope); return () -> new NormalizationLauncherWorker( connectionId, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationSummaryCheckActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationSummaryCheckActivityImpl.java index 38bf7111eb25..eec1fd1af591 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationSummaryCheckActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationSummaryCheckActivityImpl.java @@ -10,7 +10,6 @@ import datadog.trace.api.Trace; import io.airbyte.api.client.AirbyteApiClient; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.AttemptNormalizationStatusRead; import io.airbyte.api.client.model.generated.AttemptNormalizationStatusReadList; import io.airbyte.api.client.model.generated.JobIdRequestBody; @@ -48,8 +47,10 @@ public boolean shouldRunNormalization(final Long jobId, final Long attemptNumber final AttemptNormalizationStatusReadList AttemptNormalizationStatusReadList; try { - AttemptNormalizationStatusReadList = airbyteApiClient.getJobsApi().getAttemptNormalizationStatusesForJob(new JobIdRequestBody().id(jobId)); - } catch (final ApiException e) { + AttemptNormalizationStatusReadList = AirbyteApiClient.retryWithJitter( + () -> airbyteApiClient.getJobsApi().getAttemptNormalizationStatusesForJob(new JobIdRequestBody().id(jobId)), + "get normalization statuses"); + } catch (final Exception e) { throw Activity.wrap(e); } final AtomicLong totalRecordsCommitted = new AtomicLong(0L); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/PersistStateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/PersistStateActivityImpl.java index f9ed3057066c..4b6dd00753b1 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/PersistStateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/PersistStateActivityImpl.java @@ -12,7 +12,6 @@ import com.google.common.annotations.VisibleForTesting; import datadog.trace.api.Trace; import io.airbyte.api.client.AirbyteApiClient; -import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.ConnectionIdRequestBody; import io.airbyte.api.client.model.generated.ConnectionState; import io.airbyte.api.client.model.generated.ConnectionStateCreateOrUpdate; @@ -54,8 +53,10 @@ public boolean persist(final UUID connectionId, final StandardSyncOutput syncOut try { final Optional maybeStateWrapper = StateMessageHelper.getTypedState(state.getState(), featureFlags.useStreamCapableState()); if (maybeStateWrapper.isPresent()) { - final ConnectionState previousState = airbyteApiClient.getStateApi() - .getState(new ConnectionIdRequestBody().connectionId(connectionId)); + final ConnectionState previousState = + AirbyteApiClient.retryWithJitter( + () -> airbyteApiClient.getStateApi().getState(new ConnectionIdRequestBody().connectionId(connectionId)), + "get state"); if (featureFlags.needStateValidation() && previousState != null) { final StateType newStateType = maybeStateWrapper.get().getStateType(); final StateType prevStateType = convertClientStateTypeToInternal(previousState.getStateType()); @@ -65,12 +66,17 @@ public boolean persist(final UUID connectionId, final StandardSyncOutput syncOut } } - airbyteApiClient.getStateApi().createOrUpdateState( - new ConnectionStateCreateOrUpdate() - .connectionId(connectionId) - .connectionState(StateConverter.toClient(connectionId, maybeStateWrapper.orElse(null)))); + AirbyteApiClient.retryWithJitter( + () -> { + airbyteApiClient.getStateApi().createOrUpdateState( + new ConnectionStateCreateOrUpdate() + .connectionId(connectionId) + .connectionState(StateConverter.toClient(connectionId, maybeStateWrapper.orElse(null)))); + return null; + }, + "create or update state"); } - } catch (final ApiException e) { + } catch (final Exception e) { throw new RuntimeException(e); } return true; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java index b7104497e71a..e5494e8702bf 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationActivityImpl.java @@ -307,7 +307,9 @@ private CheckedSupplier, Exception> throws ApiException { final JobIdRequestBody id = new JobIdRequestBody(); id.setId(Long.valueOf(jobRunConfig.getJobId())); - final var jobInfo = airbyteApiClient.getJobsApi().getJobInfoLight(id); + final var jobInfo = AirbyteApiClient.retryWithJitter( + () -> airbyteApiClient.getJobsApi().getJobInfoLight(id), + "get job info light"); LOGGER.info("received response from from jobsApi.getJobInfoLight: {}", jobInfo); final var jobScope = jobInfo.getJob().getConfigId(); final var connectionId = UUID.fromString(jobScope); diff --git a/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java b/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java index 9eee9d0a10d1..ec472bc46895 100644 --- a/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java +++ b/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/AsyncOrchestratorPodProcessIntegrationTest.java @@ -114,7 +114,7 @@ public void testAsyncOrchestratorPodProcess(final String pullPolicy) throws Inte null, null, null, - Map.of(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"), + Map.of(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true", EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, "false"), serverPort); final Map portMap = Map.of( diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java index e1443fbbb91a..f5685c245e44 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultCheckConnectionWorkerTest.java @@ -77,7 +77,7 @@ void setup() throws IOException, WorkerException { .withConnectionStatus(new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.FAILED).withMessage("failed to connect")); failureStreamFactory = noop -> Lists.newArrayList(failureMessage).stream(); - final AirbyteMessage traceMessage = AirbyteMessageUtils.createTraceMessage("some error from the connector", 123.0); + final AirbyteMessage traceMessage = AirbyteMessageUtils.createErrorMessage("some error from the connector", 123.0); traceMessageStreamFactory = noop -> Lists.newArrayList(traceMessage).stream(); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java index ff7534073108..b9fe4417657d 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultDiscoverCatalogWorkerTest.java @@ -141,7 +141,7 @@ void testDiscoverSchemaProcessFail() throws Exception { @Test void testDiscoverSchemaProcessFailWithTraceMessage() throws Exception { final AirbyteStreamFactory traceStreamFactory = noop -> Lists.newArrayList( - AirbyteMessageUtils.createTraceMessage("some error from the connector", 123.0)).stream(); + AirbyteMessageUtils.createErrorMessage("some error from the connector", 123.0)).stream(); when(process.exitValue()).thenReturn(1); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultGetSpecWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultGetSpecWorkerTest.java index 787641266f72..7ddbdfc17d2c 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultGetSpecWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/general/DefaultGetSpecWorkerTest.java @@ -114,7 +114,7 @@ void testFailureOnNonzeroExitCode() throws InterruptedException, IOException { @Test void testFailureOnNonzeroExitCodeWithTraceMessage() throws WorkerException, InterruptedException { - final AirbyteMessage message = AirbyteMessageUtils.createTraceMessage("some error from the connector", 123.0); + final AirbyteMessage message = AirbyteMessageUtils.createErrorMessage("some error from the connector", 123.0); when(process.getInputStream()).thenReturn(new ByteArrayInputStream(Jsons.serialize(message).getBytes(Charsets.UTF_8))); when(process.waitFor(anyLong(), any())).thenReturn(true); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java index a4752364d409..4b1da61dd239 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java @@ -37,6 +37,7 @@ import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.errorreporter.JobErrorReporter; +import io.airbyte.persistence.job.factory.OAuthConfigSupplier; import io.airbyte.persistence.job.factory.SyncJobFactory; import io.airbyte.persistence.job.models.Attempt; import io.airbyte.persistence.job.models.AttemptStatus; @@ -116,6 +117,9 @@ class JobCreationAndStatusUpdateActivityTest { @Mock private StreamResetPersistence mStreamResetPersistence; + @Mock + private OAuthConfigSupplier mOAuthConfigSupplier; + @InjectMocks private JobCreationAndStatusUpdateActivityImpl jobCreationAndStatusUpdateActivity; @@ -186,6 +190,8 @@ void createResetJob() throws JsonValidationException, ConfigNotFoundException, I final JobCreationOutput output = jobCreationAndStatusUpdateActivity.createNewJob(new JobCreationInput(CONNECTION_ID)); + Mockito.verify(mOAuthConfigSupplier).injectDestinationOAuthParameters(any(), any(), any()); + Assertions.assertThat(output.getJobId()).isEqualTo(JOB_ID); } @@ -448,6 +454,16 @@ void setAttemptFailure() throws IOException { verify(mJobPersistence).writeAttemptFailureSummary(JOB_ID, ATTEMPT_ID, failureSummary); } + @Test + void setAttemptFailureManuallyTerminated() throws IOException { + jobCreationAndStatusUpdateActivity + .attemptFailure(new AttemptFailureInput(JOB_ID, ATTEMPT_ID, CONNECTION_ID, standardSyncOutput, null)); + + verify(mJobPersistence).failAttempt(JOB_ID, ATTEMPT_ID); + verify(mJobPersistence).writeOutput(JOB_ID, ATTEMPT_ID, jobOutput); + verify(mJobPersistence).writeAttemptFailureSummary(JOB_ID, ATTEMPT_ID, null); + } + @Test void setAttemptFailureWrapException() throws IOException { final Exception exception = new IOException(TEST_EXCEPTION_MESSAGE); diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 38a88c38b5fa..9f4bc12c69e4 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -92,7 +92,7 @@ class AirbytePythonPlugin implements Plugin { pip 'black:22.3.0' pip 'mypy:0.930' pip 'isort:5.6.4' - pip 'pytest:6.1.2' + pip 'pytest:6.2.5' pip 'coverage[toml]:6.3.1' } diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 30f16f593632..7f5f1aef6a17 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.40.46" +version: "0.42.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.21" +appVersion: "0.40.22" dependencies: - name: common diff --git a/charts/airbyte-connector-builder-server/.gitignore b/charts/airbyte-connector-builder-server/.gitignore new file mode 100644 index 000000000000..88e91e8a8f34 --- /dev/null +++ b/charts/airbyte-connector-builder-server/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/charts/airbyte-connector-builder-server/Chart.lock b/charts/airbyte-connector-builder-server/Chart.lock new file mode 100644 index 000000000000..746fb41b3a91 --- /dev/null +++ b/charts/airbyte-connector-builder-server/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-17T18:35:15.123937677Z" diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml new file mode 100644 index 000000000000..36add4a9a429 --- /dev/null +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -0,0 +1,31 @@ +apiVersion: v2 +name: connector-builder-server +description: Helm chart to deploy airbyte-connector-builder-server + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: "0.42.0" + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.40.22" + +dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x diff --git a/charts/airbyte-connector-builder-server/README.md b/charts/airbyte-connector-builder-server/README.md new file mode 100644 index 000000000000..1cf47631d75e --- /dev/null +++ b/charts/airbyte-connector-builder-server/README.md @@ -0,0 +1,79 @@ +# server + +![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.4](https://img.shields.io/badge/AppVersion-0.40.4-informational?style=flat-square) + +Helm chart to deploy airbyte-server + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.configMapName | string | `""` | | +| global.credVolumeOverride | string | `""` | | +| global.database.host | string | `"example.com"` | | +| global.database.port | string | `"5432"` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.extraContainers | list | `[]` | | +| global.logs.accessKey.existingSecret | string | `""` | | +| global.logs.accessKey.existingSecretKey | string | `""` | | +| global.logs.accessKey.password | string | `"minio"` | | +| global.logs.externalMinio.enabled | bool | `false` | | +| global.logs.externalMinio.host | string | `"localhost"` | | +| global.logs.externalMinio.port | int | `9000` | | +| global.logs.gcs.bucket | string | `""` | | +| global.logs.gcs.credentials | string | `""` | | +| global.logs.gcs.credentialsJson | string | `""` | | +| global.logs.minio.enabled | bool | `true` | | +| global.logs.s3.bucket | string | `"airbyte-dev-logs"` | | +| global.logs.s3.bucketRegion | string | `""` | | +| global.logs.s3.enabled | bool | `false` | | +| global.logs.secretKey.existingSecret | string | `""` | | +| global.logs.secretKey.existingSecretKey | string | `""` | | +| global.logs.secretKey.password | string | `"minio123"` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/server"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `30` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| log.level | string | `"INFO"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `10` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| service.annotations | object | `{}` | | +| service.port | int | `8001` | | +| service.type | string | `"ClusterIP"` | | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) diff --git a/charts/airbyte-connector-builder-server/templates/_helpers.tpl b/charts/airbyte-connector-builder-server/templates/_helpers.tpl new file mode 100644 index 000000000000..e34859a9d375 --- /dev/null +++ b/charts/airbyte-connector-builder-server/templates/_helpers.tpl @@ -0,0 +1,73 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} + +{{/* +Define imageTag +*/}} + +{{- define "connectorBuilderServer.imageTag" -}} +{{- if .Values.image.tag }} + {{- printf "%s" .Values.image.tag }} +{{- else if ((.Values.global.image).tag) }} + {{- printf "%s" .Values.global.image.tag }} +{{- else }} + {{- printf "%s" .Chart.AppVersion }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/airbyte-connector-builder-server/templates/deployment.yaml b/charts/airbyte-connector-builder-server/templates/deployment.yaml new file mode 100644 index 000000000000..9d3cd3777e02 --- /dev/null +++ b/charts/airbyte-connector-builder-server/templates/deployment.yaml @@ -0,0 +1,131 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + strategy: + type: Recreate # Needed due to volume claims + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-server-container + image: {{ printf "%s:%s" .Values.image.repository (include "connectorBuilderServer.imageTag" .) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + {{- end }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: server-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if .Values.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + # TODO: add a health check endpoint to connector builder server + # {{- if .Values.livenessProbe.enabled }} + # livenessProbe: + # httpGet: + # path: /api/v1/health + # port: http + # initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + # periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + # timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + # successThreshold: {{ .Values.livenessProbe.successThreshold }} + # failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + # {{- end }} + # {{- if .Values.readinessProbe.enabled }} + # readinessProbe: + # httpGet: + # path: /api/v1/health + # port: http + # initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + # periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + # timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + # successThreshold: {{ .Values.readinessProbe.successThreshold }} + # failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + # {{- end }} + + ports: + - name: http + containerPort: 80 + protocol: TCP + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} diff --git a/charts/airbyte-connector-builder-server/templates/secrets.yaml b/charts/airbyte-connector-builder-server/templates/secrets.yaml new file mode 100644 index 000000000000..e00ae6babbce --- /dev/null +++ b/charts/airbyte-connector-builder-server/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: connector-builder-server-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/charts/airbyte-connector-builder-server/templates/service.yaml b/charts/airbyte-connector-builder-server/templates/service.yaml new file mode 100644 index 000000000000..251e2394fd40 --- /dev/null +++ b/charts/airbyte-connector-builder-server/templates/service.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-connector-builder-server-svc + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} diff --git a/charts/airbyte-connector-builder-server/values.yaml b/charts/airbyte-connector-builder-server/values.yaml new file mode 100644 index 000000000000..0f385a1d7759 --- /dev/null +++ b/charts/airbyte-connector-builder-server/values.yaml @@ -0,0 +1,178 @@ +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + configMapName: "" + secretName: "" + credVolumeOverride: "" + extraContainers: [] + database: + secretName: "" + secretValue: "" + host: "example.com" + port: "5432" + +enabled: true +## replicaCount Number of server replicas +replicaCount: 1 + +## image.repository The repository to use for the airbyte server image. +## image.pullPolicy the pull policy to use for the airbyte server image +## image.tag The airbyte server image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/connector-builder-server + pullPolicy: IfNotPresent + +## podAnnotations [object] Add extra annotations to the server pod +## +podAnnotations: {} + +## containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the server containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## livenessProbe.enabled Enable livenessProbe on the server +## livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## livenessProbe.periodSeconds Period seconds for livenessProbe +## livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## livenessProbe.failureThreshold Failure threshold for livenessProbe +## livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## readinessProbe.enabled Enable readinessProbe on the server +## readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## readinessProbe.periodSeconds Period seconds for readinessProbe +## readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## readinessProbe.failureThreshold Failure threshold for readinessProbe +## readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## Server app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## resources.limits [object] The resources limits for the server container +## resources.requests [object] The requested resources for the server container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## service.type The service type to use for the API server +## service.port The service port to expose the API server on +service: + type: ClusterIP + port: 8003 + annotations: {} + +## nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## tolerations [array] Tolerations for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## affinity [object] Affinity and anti-affinity for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## log.level The log level to log at +log: + level: "INFO" + +## extraVolumeMounts [array] Additional volumeMounts for server container(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## extraVolumes [array] Additional volumes for server pod(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: AIRBYTE_VERSION +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: AIRBYTE_VERSION +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +env_vars: {} diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 220a99704197..d31414b59bba 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -21,7 +21,7 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.21" +appVersion: "0.40.22" dependencies: - name: common diff --git a/charts/airbyte-cron/templates/deployment.yaml b/charts/airbyte-cron/templates/deployment.yaml index ddade74521b5..f4456e7f19c8 100644 --- a/charts/airbyte-cron/templates/deployment.yaml +++ b/charts/airbyte-cron/templates/deployment.yaml @@ -116,6 +116,11 @@ spec: {{- end }} {{- end }} + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} securityContext: {{- toYaml .Values.containerSecurityContext | nindent 12 }} {{- end }} diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 0b47ac842a81..7ae4f5f6c52c 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.40.46" +version: "0.42.0" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 66a424d97a4a..1fc958ace54b 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.40.46" +version: "0.42.0" # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 1651052585bd..729fb23853e9 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.45.3 +version: "0.42.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.21" +appVersion: "0.40.22" dependencies: - name: common diff --git a/charts/airbyte-server/templates/deployment.yaml b/charts/airbyte-server/templates/deployment.yaml index 7f32e0aa1306..d31918e37d4b 100644 --- a/charts/airbyte-server/templates/deployment.yaml +++ b/charts/airbyte-server/templates/deployment.yaml @@ -52,6 +52,11 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: AIRBYTE_VERSION + - name: AUTO_DETECT_SCHEMA + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AUTO_DETECT_SCHEMA - name: CONFIG_ROOT valueFrom: configMapKeyRef: diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index c8e2cf451449..6b00f32a60eb 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.40.46" +version: "0.42.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.21" +appVersion: "0.40.22" dependencies: - name: common diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 1268074824be..a345868c59cb 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.40.46" +version: "0.42.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.21" +appVersion: "0.40.22" dependencies: - name: common diff --git a/charts/airbyte-webapp/templates/deployment.yaml b/charts/airbyte-webapp/templates/deployment.yaml index 52532747602e..71b2b4b981d7 100644 --- a/charts/airbyte-webapp/templates/deployment.yaml +++ b/charts/airbyte-webapp/templates/deployment.yaml @@ -55,6 +55,11 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: API_URL + - name: CONNECTOR_BUILDER_API_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_BUILDER_API_URL - name: TRACKING_STRATEGY valueFrom: configMapKeyRef: @@ -65,6 +70,11 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: INTERNAL_API_HOST + - name: CONNECTOR_BUILDER_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_BUILDER_API_HOST {{- end }} # Values from secret {{- if .Values.secrets }} diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index b32fd8e8d7ec..bdba920bbe57 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.40.46" +version: "0.42.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.21" +appVersion: "0.40.22" dependencies: - name: common diff --git a/charts/airbyte-worker/templates/deployment.yaml b/charts/airbyte-worker/templates/deployment.yaml index 066a11da6179..30b4902a914f 100644 --- a/charts/airbyte-worker/templates/deployment.yaml +++ b/charts/airbyte-worker/templates/deployment.yaml @@ -288,6 +288,11 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + - name: AUTO_DETECT_SCHEMA + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AUTO_DETECT_SCHEMA - name: USE_STREAM_CAPABLE_STATE valueFrom: configMapKeyRef: diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 15e2bb68b44d..6cd6d6acaf21 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,27 +4,30 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.40.46 + version: 0.42.0 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.40.46 + version: 0.42.0 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.40.46 + version: 0.42.0 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.40.46 + version: 0.42.0 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.40.46 + version: 0.42.0 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.40.46 + version: 0.42.0 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.40.46 + version: 0.42.0 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.40.46 -digest: sha256:f36ae5ccd39d055968406a792d76d6cc45caaea03942cd2056e5e33e487b35cb -generated: "2022-11-17T15:34:35.729309171Z" + version: 0.42.0 +- name: connector-builder-server + repository: https://airbytehq.github.io/helm-charts/ + version: 0.42.0 +digest: sha256:5158ff8ac210e556737a2b07b7093ecfb3e1b65dbccb55696b69d2713bebf386 +generated: "2022-11-22T11:01:14.830991517Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 40079a15f0be..a8a4c0e959c5 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,14 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.40.46 - +version: 0.42.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.21" +appVersion: "0.40.22" dependencies: - name: common @@ -33,33 +32,36 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: "https://airbytehq.github.io/helm-charts/" - version: 0.40.46 + version: 0.42.0 - condition: temporal.enabled name: temporal repository: "https://airbytehq.github.io/helm-charts/" - version: 0.40.46 + version: 0.42.0 - condition: webapp.enabled name: webapp repository: "https://airbytehq.github.io/helm-charts/" - version: 0.40.46 + version: 0.42.0 - condition: server.enabled name: server repository: "https://airbytehq.github.io/helm-charts/" - version: 0.40.46 + version: 0.42.0 - condition: worker.enabled name: worker repository: "https://airbytehq.github.io/helm-charts/" - version: 0.40.46 + version: 0.42.0 - condition: pod-sweeper.enabled name: pod-sweeper repository: "https://airbytehq.github.io/helm-charts/" - version: 0.40.46 + version: 0.42.0 - condition: metrics.enabled name: metrics repository: "https://airbytehq.github.io/helm-charts/" - version: 0.40.46 + version: 0.42.0 - condition: cron.enabled - name: cron + name: cron repository: "https://airbytehq.github.io/helm-charts/" - version: 0.40.46 - + version: 0.42.0 + - condition: connector-builder-server.enabled + name: connector-builder-server + repository: "https://airbytehq.github.io/helm-charts/" + version: 0.42.0 diff --git a/charts/airbyte/Chart.yaml.test b/charts/airbyte/Chart.yaml.test index 3e3861396681..d4643cc7e72e 100644 --- a/charts/airbyte/Chart.yaml.test +++ b/charts/airbyte/Chart.yaml.test @@ -65,3 +65,7 @@ dependencies: name: cron repository: "file://../airbyte-cron" version: "*" + - condition: connector-builder-server.enabled + name: connector-builder-server + repository: "file://../airbyte-connector-builder-server" + version: "*" diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index f60112cd41fd..7076774d801e 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,6 +1,6 @@ # airbyte -![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.21](https://img.shields.io/badge/AppVersion-0.40.21-informational?style=flat-square) +![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.22](https://img.shields.io/badge/AppVersion-0.40.22-informational?style=flat-square) Helm chart to deploy airbyte @@ -248,7 +248,7 @@ Helm chart to deploy airbyte | worker.hpa.enabled | bool | `false` | | | worker.image.pullPolicy | string | `"IfNotPresent"` | | | worker.image.repository | string | `"airbyte/worker"` | | -| worker.image.tag | string | `"0.40.21"` | | +| worker.image.tag | string | `"0.40.22"` | | | worker.livenessProbe.enabled | bool | `true` | | | worker.livenessProbe.failureThreshold | int | `3` | | | worker.livenessProbe.initialDelaySeconds | int | `30` | | diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 83f4115d184f..a219395275e4 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -11,6 +11,7 @@ metadata: data: AIRBYTE_VERSION: {{ .Values.version | default .Chart.AppVersion }} API_URL: {{ .Values.webapp.api.url }} + CONNECTOR_BUILDER_API_URL: {{ .Values.webapp.connectorBuilderServer.url | quote }} CONFIG_ROOT: /configs CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.35.15.001" DATA_DOCKER_MOUNT: airbyte_data @@ -22,6 +23,7 @@ data: GCS_LOG_BUCKET: {{ .Values.global.logs.gcs.bucket | quote }} GOOGLE_APPLICATION_CREDENTIALS: {{ include "airbyte.gcpLogCredentialsPath" . | quote }} INTERNAL_API_HOST: {{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }} + CONNECTOR_BUILDER_API_HOST: {{ .Release.Name }}-airbyte-connector-builder-server-svc:{{ .Values.connectorBuilderServer.service.port }} {{- if $.Values.global.jobs.kube.annotations }} JOB_KUBE_ANNOTATIONS: {{ $.Values.global.jobs.kube.annotations | include "airbyte.flattenMap" | quote }} {{- end }} @@ -61,6 +63,7 @@ data: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" WORKFLOW_FAILURE_RESTART_DELAY_SECONDS: "" USE_STREAM_CAPABLE_STATE: "true" + AUTO_DETECT_SCHEMA: "false" CONTAINER_ORCHESTRATOR_ENABLED: {{ .Values.worker.containerOrchestrator.enabled | quote }} CONTAINER_ORCHESTRATOR_IMAGE: {{ .Values.worker.containerOrchestrator.image | quote }} WORKERS_MICRONAUT_ENVIRONMENTS: "control-plane" diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 1c5ec6ccbff0..d4dbda991b4f 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -4,14 +4,11 @@ ## global -- Global params that are overwritten with umbrella chart global: - - ## serviceAccountName -- Service Account name override serviceAccountName: &service-account-name "airbyte-admin" ## deploymentMode -- Deployment mode, whether or not render the default env vars and volumes in deployment spec deploymentMode: "oss" - ## database [object] -- Object used to overrite database configuration(to use external DB) ## database.secretName -- secret name where DB creds are stored ## database.secretValue -- secret value for database password @@ -285,12 +282,13 @@ webapp: ## webapp.api.url The webapp API url. api: url: /api/v1/ + connectorBuilderServer: + url: /connector-builder-api ## webapp.fullstory.enabled Whether or not to enable fullstory fullstory: enabled: false - ## webapp.extraVolumeMounts [array] Additional volumeMounts for webapp container(s). ## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): ## extraVolumeMounts: @@ -383,10 +381,6 @@ webapp: ## DATABASE_PORT: 5432 env_vars: {} - - - - ## @section Pod Sweeper parameters pod-sweeper: @@ -591,8 +585,6 @@ server: log: level: "INFO" - - ## server.extraVolumeMounts [array] Additional volumeMounts for server container(s). ## Examples (when using `server.containerSecurityContext.readOnlyRootFilesystem=true`): ## extraVolumeMounts: @@ -1234,7 +1226,6 @@ minio: rootUser: minio rootPassword: minio123 - ## @section cron parameters cron: @@ -1332,8 +1323,6 @@ cron: log: level: "INFO" - - ## cron.extraVolumeMounts [array] Additional volumeMounts for cron container(s). ## Examples (when using `cron.containerSecurityContext.readOnlyRootFilesystem=true`): ## extraVolumeMounts: @@ -1417,4 +1406,9 @@ cron: ## env_vars: ## DATABASE_HOST: airbyte-db ## DATABASE_PORT: 5432 - env_vars: {} \ No newline at end of file + env_vars: {} + +connectorBuilderServer: + enabled: true + service: + port: 8003 diff --git a/charts/airbyte/values.yaml.test b/charts/airbyte/values.yaml.test index 57b30f1c393a..a7bde8239665 100644 --- a/charts/airbyte/values.yaml.test +++ b/charts/airbyte/values.yaml.test @@ -274,6 +274,8 @@ webapp: ## webapp.api.url The webapp API url. api: url: /api/v1/ + connectorBuilderServer: + url: /connector-builder-api ## webapp.fullstory.enabled Whether or not to enable fullstory fullstory: @@ -1393,3 +1395,8 @@ cron: ## DATABASE_HOST: airbyte-db ## DATABASE_PORT: 5432 env_vars: {} + +connectorBuilderServer: + enabled: true + service: + port: 8003 diff --git a/docker-compose.yaml b/docker-compose.yaml index 9dfaca3dc1e5..3fb296e07146 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -105,6 +105,7 @@ services: - ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS=${ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS} - ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS=${ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS} - WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=${WORKFLOW_FAILURE_RESTART_DELAY_SECONDS} + - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA} - USE_STREAM_CAPABLE_STATE=${USE_STREAM_CAPABLE_STATE} - MICRONAUT_ENVIRONMENTS=${WORKERS_MICRONAUT_ENVIRONMENTS} volumes: @@ -147,6 +148,7 @@ services: - WORKER_ENVIRONMENT=${WORKER_ENVIRONMENT} - WORKSPACE_ROOT=${WORKSPACE_ROOT} - GITHUB_STORE_BRANCH=${GITHUB_STORE_BRANCH} + - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA} ports: - 8001 volumes: @@ -166,6 +168,7 @@ services: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} - API_URL=${API_URL:-} + - CONNECTOR_BUILDER_API_URL=${CONNECTOR_BUILDER_API_URL:-} - INTERNAL_API_HOST=${INTERNAL_API_HOST} - CONNECTOR_BUILDER_API_HOST=${CONNECTOR_BUILDER_API_HOST} - OPENREPLAY=${OPENREPLAY:-} @@ -218,7 +221,7 @@ services: container_name: airbyte-connector-builder-server restart: unless-stopped ports: - - 8003:80 + - 80 environment: - AIRBYTE_VERSION=${VERSION} networks: @@ -229,6 +232,7 @@ services: ports: - 8000:8000 - 8001:8001 + - 8003:8003 environment: - BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME} - BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD} diff --git a/docs/cloud/dbt-cloud-integration.md b/docs/cloud/dbt-cloud-integration.md new file mode 100644 index 000000000000..033dc58fe8ef --- /dev/null +++ b/docs/cloud/dbt-cloud-integration.md @@ -0,0 +1,29 @@ +# Using the dbt Cloud integration + +## Step 1: Generate a service token + +Generate a [service token](https://docs.getdbt.com/docs/dbt-cloud-apis/service-tokens#generating-service-account-tokens) for your dbt Cloud transformation. + +## Step 2: Set up the dbt Cloud integration in Airbyte Cloud + +To set up the dbt Cloud integration in Airbyte Cloud: + +1. On the Airbyte Cloud dashboard, click **Settings**. + +2. Click **dbt Cloud integration**. + +3. Paste the service token from [Step 1](#step-1-prerequisites) and click **Save changes**. + +4. Click **Connections** and select the connection you want to add a dbt transformation to. + +5. Go to the **Transformation** tab and click **+ Add transformation**. + +6. Select the transformation from the dropdown and click **Save changes**. The transformation will run during the subsequent syncs until you remove it. + +:::note + +You can have multiple transformations per connection. + +::: + +8. To remove a transformation, click **X** on the transformation and click **Save changes**. diff --git a/docs/cloud/getting-started-with-airbyte-cloud.md b/docs/cloud/getting-started-with-airbyte-cloud.md index 69dd16e5edd7..32839529162b 100644 --- a/docs/cloud/getting-started-with-airbyte-cloud.md +++ b/docs/cloud/getting-started-with-airbyte-cloud.md @@ -20,8 +20,6 @@ To use Airbyte Cloud: A workspace lets you collaborate with team members and share resources across your team under a shared billing account. ::: -You will be greeted with an onboarding tutorial to help you set up your first connection. If you haven’t set up a connection on Airbyte Cloud before, we highly recommend following the tutorial. If you are familiar with the connection setup process, click **Skip Onboarding** and follow this guide to set up your next connection. - ## Set up a source :::info diff --git a/docs/connector-development/config-based/connector-builder-ui.md b/docs/connector-development/config-based/connector-builder-ui.md index 1569c50f8ea6..7beb15b3ab80 100644 --- a/docs/connector-development/config-based/connector-builder-ui.md +++ b/docs/connector-development/config-based/connector-builder-ui.md @@ -2,6 +2,10 @@ The connector builder UI provides an ergonomic iteration interface on top of the [low-code YAML format](understanding-the-yaml-file/yaml-overview). We recommend using it to iterate on your YAML connectors. +:::caution +The connector builder UI is in alpha, which means it’s still in active development and may include backward-incompatible changes. Share feedback and requests with us on our Slack channel or email us at feedback@airbyte.io +::: + ## Getting started The high level flow for using the connector builder is as follows: diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index f333cbf60a5a..a70a25283a03 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -136,7 +136,8 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| -| 1.2.7 | 2022-11-11 | [#19358](https://github.com/airbytehq/airbyte/pull/19358) | fixed check method to capture mismatch dataset location | +| 1.2.8 | 2022-11-22 | [#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage | +| 1.2.7 | 2022-11-11 | [#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location | | 1.2.6 | 2022-11-10 | [#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors | | 1.2.5 | 2022-10-19 | [#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs | | 1.2.4 | 2022-09-26 | [#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header | @@ -189,7 +190,8 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| -| 1.2.7 | 2022-11-11 | [#19358](https://github.com/airbytehq/airbyte/pull/19358) | fixed check method to capture mismatch dataset location | +| 1.2.8 | 2022-11-22 | [#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage | +| 1.2.7 | 2022-11-11 | [#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location | | 1.2.6 | 2022-11-10 | [#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors | | 1.2.5 | 2022-10-19 | [#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs | | 1.2.4 | 2022-09-26 | [#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header | diff --git a/docs/integrations/sources/asana.md b/docs/integrations/sources/asana.md index 757b6bfa4a68..46b0e88c3663 100644 --- a/docs/integrations/sources/asana.md +++ b/docs/integrations/sources/asana.md @@ -66,6 +66,7 @@ The connector is restricted by normal Asana [requests limitation](https://develo | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------- | +| 0.1.5 | 2022-11-16 | [19561](https://github.com/airbytehq/airbyte/pull/19561) | Added errors handling, updated SAT with new format | 0.1.4 | 2022-08-18 | [15749](https://github.com/airbytehq/airbyte/pull/15749) | Add cache to project stream | | 0.1.3 | 2021-10-06 | [6832](https://github.com/airbytehq/airbyte/pull/6832) | Add oauth init flow parameters support | | 0.1.2 | 2021-09-24 | [6402](https://github.com/airbytehq/airbyte/pull/6402) | Fix SAT tests: update schemas and invalid\_config.json file | diff --git a/docs/integrations/sources/chargebee.md b/docs/integrations/sources/chargebee.md index 6c1f089e9ddb..98d0e65783ef 100644 --- a/docs/integrations/sources/chargebee.md +++ b/docs/integrations/sources/chargebee.md @@ -13,7 +13,7 @@ To set up the Chargebee source connector, you'll need the [Chargebee API key](ht 3. On the Set up the source page, select **Chargebee** from the Source type dropdown. 4. Enter the name for the Chargebee connector. 5. For **Site**, enter the site prefix for your Chargebee instance. -6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +6. For **Start Date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. 7. For **API Key**, enter the [Chargebee API key](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2#api_authentication). 8. For **Product Catalog**, enter the Chargebee [Product Catalog version](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2). 9. Click **Set up source**. diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 671ded370b43..eed6a487b7cd 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -21,13 +21,13 @@ To set up Facebook Marketing as a source in Airbyte Cloud: 3. On the Set up the source page, select **Facebook Marketing** from the **Source type** dropdown. 4. For Name, enter a name for the Facebook Marketing connector. 5. Click **Authenticate your account** to authorize your [Meta for Developers](https://developers.facebook.com/) account. Airbyte will authenticate the account you are already logged in to. Make sure you are logged into the right account. -6. For **Start Date**, enter the date in the YYYY-MM-DDTHR:MIN:S format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. +6. For **Start Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. :::warning Insight tables are only able to pull data from 37 months. If you are syncing insight tables and your start date is older than 37 months, your sync will fail. ::: -7. For **End Date**, enter the date in the YYYY-MM-DDTHR:MIN:S format. The data added on and before this date will be replicated. If this field is blank, Airbyte will replicate the latest data. +7. For **End Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and before this date will be replicated. If this field is blank, Airbyte will replicate the latest data. 8. For Account ID, enter your [Facebook Ad Account ID Number](https://www.facebook.com/business/help/1492627900875762). 9. (Optional) Toggle the **Include Deleted** button to include data from deleted Campaigns, Ads, and AdSets. @@ -45,9 +45,9 @@ To set up Facebook Marketing as a source in Airbyte Cloud: 1. For **Name**, enter a name for the insight. This will be used as the Airbyte stream name 2. For **Fields**, enter a list of the fields you want to pull from the Facebook Marketing API. - 3. For **End Date**, enter the date in the YYYY-MM-DDTHR:MIN:S format. The data added on and before this date will be replicated. If this field is blank, Airbyte will replicate the latest data. + 3. For **End Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and before this date will be replicated. If this field is blank, Airbyte will replicate the latest data. 4. For **Breakdowns**, enter a list of the breakdowns you want to configure. - 5. For **Start Date**, enter the date in the YYYY-MM-DDTHR:MIN:S format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. + 5. For **Start Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. 6. For **Time Increment**, enter the number of days over which you want to aggregate statistics. For example, if you set this value to 7, Airbyte will report statistics as 7-day aggregates starting from the Start Date. Suppose the start and end dates are October 1st and October 30th, then the connector will output 5 records: 01 - 06, 07 - 13, 14 - 20, 21 - 27, and 28 - 30 (3 days only). @@ -127,6 +127,8 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.74 | 2022-11-25 | [19803](https://github.com/airbytehq/airbyte/pull/19803) | New default for `action_breakdowns`, improve "check" command speed | +| 0.2.73 | 2022-11-21 | [19645](https://github.com/airbytehq/airbyte/pull/19645) | Check "breakdowns" combinations | | 0.2.72 | 2022-11-04 | [18971](https://github.com/airbytehq/airbyte/pull/18971) | handle FacebookBadObjectError for empty results on async jobs | | 0.2.71 | 2022-10-31 | [18734](https://github.com/airbytehq/airbyte/pull/18734) | Reduce request record limit on retry | | 0.2.70 | 2022-10-26 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Upgrade FB SDK to v15.0 | diff --git a/docs/integrations/sources/freshdesk.md b/docs/integrations/sources/freshdesk.md index f808d5e33d92..a26743f79c96 100644 --- a/docs/integrations/sources/freshdesk.md +++ b/docs/integrations/sources/freshdesk.md @@ -14,7 +14,7 @@ To set up the Freshdesk source connector, you'll need the Freshdesk [domain URL] 4. Enter the name for the Freshdesk connector. 5. For **Domain**, enter your [Freshdesk domain URL](https://support.freshdesk.com/en/support/solutions/articles/50000004704-customizing-your-helpdesk-url). 6. For **API Key**, enter your [Freshdesk API key](https://support.freshdesk.com/support/solutions/articles/215517). -7. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +7. For **Start Date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. 8. For **Requests per minute**, enter the number of requests per minute that this source allowed to use. The Freshdesk rate limit is 50 requests per minute per app per account. 9. Click **Set up source**. diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index 621f1a7686aa..b93f275a3eba 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -130,7 +130,8 @@ Due to a limitation in the Google Ads API which does not allow getting performan | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| -| `0.2.4` | 2022-11-09 | [19208](https://github.com/airbytehq/airbyte/pull/19208) | Add TypeTransofrmer to Campaings stream to force proper type casting | +| `0.2.5` | 2022-11-22 | [19700](https://github.com/airbytehq/airbyte/pull/19700) | Fix schema for `campaigns` stream | +| `0.2.4` | 2022-11-09 | [19208](https://github.com/airbytehq/airbyte/pull/19208) | Add TypeTransofrmer to Campaings stream to force proper type casting | | `0.2.3` | 2022-10-17 | [18069](https://github.com/airbytehq/airbyte/pull/18069) | Add `segments.hour`, `metrics.ctr`, `metrics.conversions` and `metrics.conversions_values` fields to `campaigns` report stream | | `0.2.2` | 2022-10-21 | [17412](https://github.com/airbytehq/airbyte/pull/17412) | Release with CDK >= 0.2.2 | | `0.2.1` | 2022-09-29 | [17412](https://github.com/airbytehq/airbyte/pull/17412) | Always use latest CDK version | diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index 89c5163e5d76..a912ec1195b6 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -16,7 +16,7 @@ To set up the Harvest source connector, you'll need the [Harvest Account ID and 3. On the Set up the source page, select **Harvest** from the Source type dropdown. 4. Enter the name for the Harvest connector. 5. Enter your [Harvest Account ID](https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/). -6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +6. For **Start Date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. 7. For Authentication mechanism, select **Authenticate via Harvest (OAuth)** from the dropdown and click **Authenticate your Harvest account**. Log in and authorize your Harvest account. 8. Click **Set up source**. @@ -29,7 +29,7 @@ To set up the Harvest source connector, you'll need the [Harvest Account ID and 3. On the Set up the source page, select **Harvest** from the Source type dropdown. 4. Enter the name for the Harvest connector. 5. Enter your [Harvest Account ID](https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/). -6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +6. For **Start Date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. 7. For **Authentication mechanism**, select **Authenticate with Personal Access Token** from the dropdown. Enter your [Personal Access Token](https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#personal-access-tokens). 8. Click **Set up source**. diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 20e77e10e077..f1027257d46d 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -31,7 +31,7 @@ Generate access tokens with the following permissions: 4. Enter a name for your source. 5. Click **Authenticate your Instagram account**. 6. Log in and authorize the Instagram account. -7. Enter the **Start Date** in YYYY-MM-DDT00:00:00Z format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. +7. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. 8. Click **Set up source**. @@ -44,7 +44,7 @@ Generate access tokens with the following permissions: 4. Enter a name for your source. 5. Click **Authenticate your Instagram account**. 6. Log in and authorize the Instagram account. -7. Enter the **Start Date** in YYYY-MM-DDT00:00:00Z format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. +7. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. 8. Paste the access tokens from [Step 1](#step-1-set-up-instagram​). 9. Click **Set up source**. diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index b71d93c1ee75..5fc094849061 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -13,7 +13,7 @@ To set up the Iterable source connector, you'll need the Iterable [`Server-side` 3. On the Set up the source page, select **Iterable** from the Source type dropdown. 4. Enter the name for the Iterable connector. 5. For **API Key**, enter the [Iterable API key](https://support.iterable.com/hc/en-us/articles/360043464871-API-Keys-). -6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +6. For **Start Date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. 7. Click **Set up source**. ## Supported sync modes diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 32b4aac08fe4..2ce57e435ead 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -3,7 +3,7 @@ ## Features | Feature | Supported | Notes | -|:------------------------------|:----------|:----------------------------------| +| :---------------------------- | :-------- | :-------------------------------- | | Full Refresh Sync | Yes | | | Incremental - Append Sync | Yes | | | Replicate Incremental Deletes | Yes | | @@ -24,7 +24,6 @@ To solve this issue add `enabledTLSProtocols=TLSv1.2` in the JDBC parameters. Another error that users have reported when trying to connect to Amazon RDS MySQL is `Error: HikariPool-1 - Connection is not available, request timed out after 30001ms.`. Many times this is can be due to the VPC not allowing public traffic, however, we recommend going through [this AWS troubleshooting checklist](https://aws.amazon.com/premiumsupport/knowledge-center/rds-cannot-connect/) to the correct permissions/settings have been granted to allow connection to your database. - ## Getting Started \(Airbyte Cloud\) On Airbyte Cloud, only TLS connections to your MySQL instance are supported. Other than that, you can proceed with the open-source instructions below. @@ -74,15 +73,15 @@ Your database user should now be ready for use with Airbyte. ## Change Data Capture \(CDC\) -* If you need a record of deletions and can accept the limitations posted below, you should be able to use CDC for MySQL. -* If your data set is small, and you just want snapshot of your table in the destination, consider using Full Refresh replication for your table instead of CDC. -* If the limitations prevent you from using CDC and your goal is to maintain a snapshot of your table in the destination, consider using non-CDC incremental and occasionally reset the data and re-sync. -* If your table has a primary key but doesn't have a reasonable cursor field for incremental syncing \(i.e. `updated_at`\), CDC allows you to sync your table incrementally. +- If you need a record of deletions and can accept the limitations posted below, you should be able to use CDC for MySQL. +- If your data set is small, and you just want snapshot of your table in the destination, consider using Full Refresh replication for your table instead of CDC. +- If the limitations prevent you from using CDC and your goal is to maintain a snapshot of your table in the destination, consider using non-CDC incremental and occasionally reset the data and re-sync. +- If your table has a primary key but doesn't have a reasonable cursor field for incremental syncing \(i.e. `updated_at`\), CDC allows you to sync your table incrementally. #### CDC Limitations -* Make sure to read our [CDC docs](../../understanding-airbyte/cdc.md) to see limitations that impact all databases using CDC replication. -* Our CDC implementation uses at least once delivery for all change records. +- Make sure to read our [CDC docs](../../understanding-airbyte/cdc.md) to see limitations that impact all databases using CDC replication. +- Our CDC implementation uses at least once delivery for all change records. **1. Enable binary logging** @@ -96,18 +95,18 @@ binlog_row_image = FULL binlog_expire_log_seconds = 864000 ``` -* server-id : The value for the server-id must be unique for each server and replication client in the MySQL cluster. The `server-id` should be a non-zero value. If the `server-id` is already set to a non-zero value, you don't need to make any change. You can set the `server-id` to any value between 1 and 4294967295. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options.html#sysvar_server_id) -* log\_bin : The value of log\_bin is the base name of the sequence of binlog files. If the `log_bin` is already set, you don't need to make any change. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#option_mysqld_log-bin) -* binlog\_format : The `binlog_format` must be set to `ROW`. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#sysvar_binlog_format) -* binlog\_row\_image : The `binlog_row_image` must be set to `FULL`. It determines how row images are written to the binary log. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html#sysvar_binlog_row_image) -* binlog_expire_log_seconds : This is the number of seconds for automatic binlog file removal. We recommend 864000 seconds (10 days) so that in case of a failure in sync or if the sync is paused, we still have some bandwidth to start from the last point in incremental sync. We also recommend setting frequent syncs for CDC. +- server-id : The value for the server-id must be unique for each server and replication client in the MySQL cluster. The `server-id` should be a non-zero value. If the `server-id` is already set to a non-zero value, you don't need to make any change. You can set the `server-id` to any value between 1 and 4294967295. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options.html#sysvar_server_id) +- log_bin : The value of log_bin is the base name of the sequence of binlog files. If the `log_bin` is already set, you don't need to make any change. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#option_mysqld_log-bin) +- binlog_format : The `binlog_format` must be set to `ROW`. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#sysvar_binlog_format) +- binlog_row_image : The `binlog_row_image` must be set to `FULL`. It determines how row images are written to the binary log. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html#sysvar_binlog_row_image) +- binlog_expire_log_seconds : This is the number of seconds for automatic binlog file removal. We recommend 864000 seconds (10 days) so that in case of a failure in sync or if the sync is paused, we still have some bandwidth to start from the last point in incremental sync. We also recommend setting frequent syncs for CDC. **2. Enable GTIDs \(Optional\)** Global transaction identifiers \(GTIDs\) uniquely identify transactions that occur on a server within a cluster. Though not required for a Airbyte MySQL connector, using GTIDs simplifies replication and enables you to more easily confirm if primary and replica servers are consistent. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-gtids.html#option_mysqld_gtid-mode) -* Enable gtid\_mode : Boolean that specifies whether GTID mode of the MySQL server is enabled or not. Enable it via `mysql> gtid_mode=ON` -* Enable enforce\_gtid\_consistency : Boolean that specifies whether the server enforces GTID consistency by allowing the execution of statements that can be logged in a transactionally safe manner. Required when using GTIDs. Enable it via `mysql> enforce_gtid_consistency=ON` +- Enable gtid_mode : Boolean that specifies whether GTID mode of the MySQL server is enabled or not. Enable it via `mysql> gtid_mode=ON` +- Enable enforce_gtid_consistency : Boolean that specifies whether the server enforces GTID consistency by allowing the execution of statements that can be logged in a transactionally safe manner. Required when using GTIDs. Enable it via `mysql> enforce_gtid_consistency=ON` **3. Set up initial waiting time\(Optional\)** @@ -134,7 +133,6 @@ In CDC mode, the MySQl connector may need a timezone configured if the existing In this case, you can configure the server timezone to the equivalent IANA timezone compliant timezone. (e.g. CEST -> Europe/Berlin). - **Note** When a sync runs for the first time using CDC, Airbyte performs an initial consistent snapshot of your database. Airbyte doesn't acquire any table locks \(for tables defined with MyISAM engine, the tables would still be locked\) while creating the snapshot to allow writes by other database clients. But in order for the sync to work without any error/unexpected behaviour, it is assumed that no schema changes are happening while the snapshot is running. @@ -149,11 +147,12 @@ Using this feature requires additional configuration, when creating the source. 1. Configure all fields for the source as you normally would, except `SSH Tunnel Method`. 2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`. + 1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\). 2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel. :::warning - Since Airbyte Cloud requires encrypted communication, select **SSH Key Authentication** or **Password Authentication** if you selected **preferred** as the **SSL Mode**; otherwise, the connection will fail. + Since Airbyte Cloud requires encrypted communication, select **SSH Key Authentication** or **Password Authentication** if you selected **preferred** as the **SSL Mode**; otherwise, the connection will fail. ::: 3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address. @@ -177,7 +176,7 @@ This produces the private key in pem format, and the public key remains in the s MySQL data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test! | MySQL Type | Resulting Type | Notes | -|:------------------------------------------|:-----------------------|:---------------------------------------------------------------------------------------------------------------| +| :---------------------------------------- | :--------------------- | :------------------------------------------------------------------------------------------------------------- | | `bit(1)` | boolean | | | `bit(>1)` | base64 binary string | | | `boolean` | boolean | | @@ -217,8 +216,9 @@ MySQL data types are mapped to the following data types when synchronizing data. If you do not see a type in this list, assume that it is coerced into a string. We are happy to take feedback on preferred mappings. ## Upgrading from 0.6.8 and older versions to 0.6.9 and later versions + There is a backwards incompatible spec change between MySQL Source connector versions 0.6.8 and 0.6.9. As part of that spec change -`replication_method` configuration parameter was changed to `object` from `string`. +`replication_method` configuration parameter was changed to `object` from `string`. In MySQL source connector versions 0.6.8 and older, `replication_method` configuration parameter was saved in the configuration database as follows: @@ -227,6 +227,7 @@ In MySQL source connector versions 0.6.8 and older, `replication_method` configu ``` Starting with version 0.6.9, `replication_method` configuration parameter is saved as follows: + ``` "replication_method": { "method": "STANDARD" @@ -238,20 +239,24 @@ in Airbyte database. To do so, you need to run the following two SQL queries. Fo run SQL queries on Airbyte database. If you have connections with MySQL Source using _Standard_ replication method, run this SQL: + ```sql -update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "STANDARD"}', true) +update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "STANDARD"}', true) WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configuration->>'replication_method' = 'STANDARD'); ``` If you have connections with MySQL Source using _Logicai Replication (CDC)_ method, run this SQL: + ```sql -update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "CDC"}', true) +update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "CDC"}', true) WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configuration->>'replication_method' = 'CDC'); ``` ## Changelog + | Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| +| :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------- | +| 1.0.14 | 2022-11-22 | [19514](https://github.com/airbytehq/airbyte/pull/19514) | Adjust batch selection memory limits databases. | | 1.0.13 | 2022-11-14 | [18956](https://github.com/airbytehq/airbyte/pull/18956) | Clean up Tinyint Unsigned data type identification | | 1.0.12 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | | 1.0.11 | 2022-11-03 | [18851](https://github.com/airbytehq/airbyte/pull/18851) | Fix bug with unencrypted CDC connections | @@ -307,7 +312,7 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura | 0.4.6 | 2021-09-29 | [6510](https://github.com/airbytehq/airbyte/pull/6510) | Support SSL connection | | 0.4.5 | 2021-09-17 | [6146](https://github.com/airbytehq/airbyte/pull/6146) | Added option to connect to DB via SSH | | 0.4.1 | 2021-07-23 | [4956](https://github.com/airbytehq/airbyte/pull/4956) | Fix log link | -| 0.3.7 | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE\_ENTRYPOINT for Kubernetes support | +| 0.3.7 | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE_ENTRYPOINT for Kubernetes support | | 0.3.6 | 2021-06-09 | [3966](https://github.com/airbytehq/airbyte/pull/3966) | Fix excessive logging for CDC method | | 0.3.5 | 2021-06-07 | [3890](https://github.com/airbytehq/airbyte/pull/3890) | Fix CDC handle tinyint\(1\) and boolean types | | 0.3.4 | 2021-06-04 | [3846](https://github.com/airbytehq/airbyte/pull/3846) | Fix max integer value failure | @@ -328,4 +333,3 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura | 0.1.6 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySQL to use new JdbcSource | | 0.1.5 | 2020-12-11 | [1267](https://github.com/airbytehq/airbyte/pull/1267) | Support incremental sync | | 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | - diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index b031df55df84..942ed970f92a 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -35,7 +35,7 @@ You must be the owner of a Notion workspace to create a new integration. * If you select **Access Token**, paste the access token from [Step 8](#step-1-set-up-notion​). * If you select **OAuth2.0** authorization, click **Authenticate your Notion account**. * Log in and Authorize the Notion account. Select the permissions you want to allow Airbyte. -6. Enter the **Start Date** in YYYY-MM-DDT00:00:00Z format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. +6. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. 7. Click **Set up source**. @@ -49,7 +49,7 @@ You must be the owner of a Notion workspace to create a new integration. 5. Choose the method of authentication: * If you select **Access Token**, paste the access token from [Step 8](#step-1-set-up-notion​). * If you select **OAuth2.0** authorization, paste the client ID, access token, and client secret from [Step 8](#step-1-set-up-notion​). -6. Enter the **Start Date** in YYYY-MM-DDT00:00:00Z format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. +6. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. 7. Click **Set up source**. diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 76d88f61f324..c47c8ac6e69f 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -2,7 +2,7 @@ This page contains the setup guide and reference information for the Postgres source connector for CDC and non-CDC workflows. -:::note +:::note The Postgres source performs best on small databases (under 100GB). ::: @@ -85,6 +85,7 @@ The workaround for partial table syncing is to create a view on the specific col ``` CREATE VIEW as SELECT FROM ; ``` + ``` GRANT SELECT ON TABLE IN SCHEMA to ; ``` @@ -100,24 +101,25 @@ This issue is tracked in [#9771](https://github.com/airbytehq/airbyte/issues/977 4. Enter a name for your source. 5. For the **Host**, **Port**, and **DB Name**, enter the hostname, port number, and name for your Postgres database. 6. List the **Schemas** you want to sync. - :::note - The schema names are case sensitive. The 'public' schema is set by default. Multiple schemas may be used at one time. No schemas set explicitly - will sync all of existing. - ::: + :::note + The schema names are case sensitive. The 'public' schema is set by default. Multiple schemas may be used at one time. No schemas set explicitly - will sync all of existing. + ::: 7. For **User** and **Password**, enter the username and password you created in [Step 1](#step-1-optional-create-a-dedicated-read-only-user). 8. To customize the JDBC connection beyond common options, specify additional supported [JDBC URL parameters](https://jdbc.postgresql.org/documentation/head/connect.html) as key-value pairs separated by the symbol & in the **JDBC URL Parameters (Advanced)** field. - Example: key1=value1&key2=value2&key3=value3 + Example: key1=value1&key2=value2&key3=value3 - These parameters will be added at the end of the JDBC URL that the AirByte will use to connect to your Postgres database. + These parameters will be added at the end of the JDBC URL that the AirByte will use to connect to your Postgres database. - The connector now supports `connectTimeout` and defaults to 60 seconds. Setting connectTimeout to 0 seconds will set the timeout to the longest time available. + The connector now supports `connectTimeout` and defaults to 60 seconds. Setting connectTimeout to 0 seconds will set the timeout to the longest time available. - **Note:** Do not use the following keys in JDBC URL Params field as they will be overwritten by Airbyte: - `currentSchema`, `user`, `password`, `ssl`, and `sslmode`. + **Note:** Do not use the following keys in JDBC URL Params field as they will be overwritten by Airbyte: + `currentSchema`, `user`, `password`, `ssl`, and `sslmode`. + + :::warning + This is an advanced configuration option. Users are advised to use it with caution. + ::: - :::warning - This is an advanced configuration option. Users are advised to use it with caution. - ::: 9. For Airbyte Open Source, toggle the switch to connect using SSL. For Airbyte Cloud uses SSL by default. 10. For SSL Modes, select: - **disable** to disable encrypted communication between Airbyte and the source @@ -128,16 +130,16 @@ This issue is tracked in [#9771](https://github.com/airbytehq/airbyte/issues/977 - **verify-full** to always require encryption and verify the identity of the source 11. For Replication Method, select Standard or [Logical CDC](https://www.postgresql.org/docs/10/logical-replication.html) from the dropdown. Refer to [Configuring Postgres connector with Change Data Capture (CDC)](#configuring-postgres-connector-with-change-data-capture-cdc) for more information. 12. For SSH Tunnel Method, select: + - **No Tunnel** for a direct connection to the database - **SSH Key Authentication** to use an RSA Private as your secret for establishing the SSH tunnel - **Password Authentication** to use a password as your secret for establishing the SSH tunnel - + :::warning Since Airbyte Cloud requires encrypted communication, select **SSH Key Authentication** or **Password Authentication** if you selected **disable**, **allow**, or **prefer** as the **SSL Mode**; otherwise, the connection will fail. ::: - Refer to [Connect via SSH Tunnel](#connect-via-ssh-tunnel​) for more information. -13. Click **Set up source**. +Refer to [Connect via SSH Tunnel](#connect-via-ssh-tunnel​) for more information. 13. Click **Set up source**. ### Connect via SSH Tunnel​ @@ -148,16 +150,17 @@ When using an SSH tunnel, you are configuring Airbyte to connect to an intermedi To connect to a Postgres instance via an SSH tunnel: 1. While [setting up](#setup-guide) the Postgres source connector, from the SSH tunnel dropdown, select: - - SSH Key Authentication to use an RSA Private as your secret for establishing the SSH tunnel - - Password Authentication to use a password as your secret for establishing the SSH Tunnel + - SSH Key Authentication to use an RSA Private as your secret for establishing the SSH tunnel + - Password Authentication to use a password as your secret for establishing the SSH Tunnel 2. For **SSH Tunnel Jump Server Host**, enter the hostname or IP address for the intermediate (bastion) server that Airbyte will connect to. 3. For **SSH Connection Port**, enter the port on the bastion server. The default port for SSH connections is 22. 4. For **SSH Login Username**, enter the username to use when connecting to the bastion server. **Note:** This is the operating system username and not the Postgres username. 5. For authentication: - - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [RSA Private Key](#generating-an-rsa-private-key​) that you are using to create the SSH connection. - - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the Postgres password. + - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [RSA Private Key](#generating-an-rsa-private-key​) that you are using to create the SSH connection. + - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the Postgres password. #### Generating an RSA Private Key​ + The connector expects an RSA key in PEM format. To generate this key, run: ``` @@ -190,10 +193,10 @@ Airbyte requires a replication slot configured only for its use. Only one source To enable logical replication on bare metal, VMs (EC2/GCE/etc), or Docker, configure the following parameters in the [postgresql.conf file](https://www.postgresql.org/docs/current/config-setting.html) for your Postgres database: -| Parameter | Description | Set value to | -|------------|--------------|--------------| -| wal_level | Type of coding used within the Postgres write-ahead log | logical | -| max_wal_senders | The maximum number of processes used for handling WAL changes | Min: 1 | +| Parameter | Description | Set value to | +| --------------------- | ------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------- | +| wal_level | Type of coding used within the Postgres write-ahead log | logical | +| max_wal_senders | The maximum number of processes used for handling WAL changes | Min: 1 | | max_replication_slots | The maximum number of replication slots that are allowed to stream WAL changes | 1 (if Airbyte is the only service reading subscribing to WAL changes. More than 1 if other services are also reading from the WAL) | To enable logical replication on AWS Postgres RDS or Aurora​: @@ -205,11 +208,12 @@ To enable logical replication on AWS Postgres RDS or Aurora​: To enable logical replication on Azure Database for Postgres​: -Change the replication mode of your Postgres DB on Azure to `logical` using the **Replication** menu of your PostgreSQL instance in the Azure Portal. Alternatively, use the Azure CLI to run the following command: +Change the replication mode of your Postgres DB on Azure to `logical` using the **Replication** menu of your PostgreSQL instance in the Azure Portal. Alternatively, use the Azure CLI to run the following command: ``` az postgres server configuration set --resource-group group --server-name server --name azure.replication_support --value logical ``` + ``` az postgres server restart --resource-group group --name server ``` @@ -237,13 +241,17 @@ SELECT pg_create_logical_replication_slot('airbyte_slot', 'wal2json'); For each table you want to replicate with CDC, add the replication identity (the method of distinguishing between rows) first: To use primary keys to distinguish between rows for tables that don't have a large amount of data per row, run: + ``` ALTER TABLE tbl1 REPLICA IDENTITY DEFAULT; ``` + In case your tables use data types that support [TOAST](https://www.postgresql.org/docs/current/storage-toast.html) and have very large field values, use: + ``` ALTER TABLE tbl1 REPLICA IDENTITY FULL; ``` + After setting the replication identity, run: ``` @@ -311,54 +319,54 @@ The Postgres source connector supports the following [sync modes](https://docs.a According to Postgres [documentation](https://www.postgresql.org/docs/14/datatype.html), Postgres data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test! -| Postgres Type | Resulting Type | Notes | -|:--------------------------------------|:---------------|:------------------------------------------------------------------------------------------------------------------------------------------------| -| `bigint` | number | | -| `bigserial`, `serial8` | number | | -| `bit` | string | Fixed-length bit string (e.g. "0100"). | -| `bit varying`, `varbit` | string | Variable-length bit string (e.g. "0100"). | -| `boolean`, `bool` | boolean | | -| `box` | string | | -| `bytea` | string | Variable length binary string with hex output format prefixed with "\x" (e.g. "\x6b707a"). | -| `character`, `char` | string | | -| `character varying`, `varchar` | string | | -| `cidr` | string | | -| `circle` | string | | -| `date` | string | Parsed as ISO8601 date time at midnight. CDC mode doesn't support era indicators. Issue: [#14590](https://github.com/airbytehq/airbyte/issues/14590) | -| `double precision`, `float`, `float8` | number | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902). | -| `hstore` | string | | -| `inet` | string | | -| `integer`, `int`, `int4` | number | | -| `interval` | string | | -| `json` | string | | -| `jsonb` | string | | -| `line` | string | | -| `lseg` | string | | -| `macaddr` | string | | -| `macaddr8` | string | | -| `money` | number | | -| `numeric`, `decimal` | number | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902). | -| `path` | string | | -| `pg_lsn` | string | | -| `point` | string | | -| `polygon` | string | | -| `real`, `float4` | number | | -| `smallint`, `int2` | number | | -| `smallserial`, `serial2` | number | | -| `serial`, `serial4` | number | | -| `text` | string | | -| `time` | string | Parsed as a time string without a time-zone in the ISO-8601 calendar system. | -| `timetz` | string | Parsed as a time string with time-zone in the ISO-8601 calendar system. | -| `timestamp` | string | Parsed as a date-time string without a time-zone in the ISO-8601 calendar system. | -| `timestamptz` | string | Parsed as a date-time string with time-zone in the ISO-8601 calendar system. | -| `tsquery` | string | | -| `tsvector` | string | | -| `uuid` | string | | -| `xml` | string | | -| `enum` | string | | -| `tsrange` | string | | -| `array` | array | E.g. "[\"10001\",\"10002\",\"10003\",\"10004\"]". | -| composite type | string | | +| Postgres Type | Resulting Type | Notes | +| :------------------------------------ | :------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bigint` | number | | +| `bigserial`, `serial8` | number | | +| `bit` | string | Fixed-length bit string (e.g. "0100"). | +| `bit varying`, `varbit` | string | Variable-length bit string (e.g. "0100"). | +| `boolean`, `bool` | boolean | | +| `box` | string | | +| `bytea` | string | Variable length binary string with hex output format prefixed with "\x" (e.g. "\x6b707a"). | +| `character`, `char` | string | | +| `character varying`, `varchar` | string | | +| `cidr` | string | | +| `circle` | string | | +| `date` | string | Parsed as ISO8601 date time at midnight. CDC mode doesn't support era indicators. Issue: [#14590](https://github.com/airbytehq/airbyte/issues/14590) | +| `double precision`, `float`, `float8` | number | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902). | +| `hstore` | string | | +| `inet` | string | | +| `integer`, `int`, `int4` | number | | +| `interval` | string | | +| `json` | string | | +| `jsonb` | string | | +| `line` | string | | +| `lseg` | string | | +| `macaddr` | string | | +| `macaddr8` | string | | +| `money` | number | | +| `numeric`, `decimal` | number | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902). | +| `path` | string | | +| `pg_lsn` | string | | +| `point` | string | | +| `polygon` | string | | +| `real`, `float4` | number | | +| `smallint`, `int2` | number | | +| `smallserial`, `serial2` | number | | +| `serial`, `serial4` | number | | +| `text` | string | | +| `time` | string | Parsed as a time string without a time-zone in the ISO-8601 calendar system. | +| `timetz` | string | Parsed as a time string with time-zone in the ISO-8601 calendar system. | +| `timestamp` | string | Parsed as a date-time string without a time-zone in the ISO-8601 calendar system. | +| `timestamptz` | string | Parsed as a date-time string with time-zone in the ISO-8601 calendar system. | +| `tsquery` | string | | +| `tsvector` | string | | +| `uuid` | string | | +| `xml` | string | | +| `enum` | string | | +| `tsrange` | string | | +| `array` | array | E.g. "[\"10001\",\"10002\",\"10003\",\"10004\"]". | +| composite type | string | | ## Limitations @@ -366,7 +374,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp - The Postgres source connector does not alter the schema present in your database. Depending on the destination connected to this source, however, the schema may be altered. See the destination's documentation for more details. - The following two schema evolution actions are currently supported: - Adding/removing tables without resetting the entire connection at the destination - Caveat: In the CDC mode, adding a new table to a connection may become a temporary bottleneck. When a new table is added, the next sync job takes a full snapshot of the new table before it proceeds to handle any changes. + Caveat: In the CDC mode, adding a new table to a connection may become a temporary bottleneck. When a new table is added, the next sync job takes a full snapshot of the new table before it proceeds to handle any changes. - Resetting a single table within the connection without resetting the rest of the destination tables in that connection - Changing a column data type or removing a column might break connections. @@ -377,6 +385,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp When the connector is reading from a Postgres replica that is configured as a Hot Standby, any update from the primary server will terminate queries on the replica after a certain amount of time, default to 30 seconds. This default waiting time is not enough to sync any meaning amount of data. See the `Handling Query Conflicts` section in the Postgres [documentation](https://www.postgresql.org/docs/14/hot-standby.html#HOT-STANDBY-CONFLICT) for detailed explanation. Here is the typical exception: + ``` Caused by: org.postgresql.util.PSQLException: FATAL: terminating connection due to conflict with recovery Detail: User query might have needed to see row versions that must be removed. @@ -384,6 +393,7 @@ Caused by: org.postgresql.util.PSQLException: FATAL: terminating connection due ``` Possible solutions include: + - [Recommended] Set [`hot_standby_feedback`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-HOT-STANDBY-FEEDBACK) to `true` on the replica server. This parameter will prevent the primary server from deleting the write-ahead logs when the replica is busy serving user queries. However, the downside is that the write-ahead log will increase in size. - [Recommended] Sync data when there is no update running in the primary server, or sync data from the primary server. - [Not Recommended] Increase [`max_standby_archive_delay`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-MAX-STANDBY-ARCHIVE-DELAY) and [`max_standby_streaming_delay`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-MAX-STANDBY-STREAMING-DELAY) to be larger than the amount of time needed to complete the data sync. However, it is usually hard to tell how much time it will take to sync all the data. This approach is not very practical. @@ -395,6 +405,7 @@ Normally under the CDC mode, the Postgres source will first run a full refresh s > Saved offset is before Replication slot's confirmed_flush_lsn, Airbyte will trigger sync from scratch The root causes is that the WALs needed for the incremental sync has been removed by Postgres. This can occur under the following scenarios: + - When there are lots of database updates resulting in more WAL files than allowed in the `pg_wal` directory, Postgres will purge or archive the WAL files. This scenario is preventable. Possible solutions include: - Sync the data source more frequently. The downside is that more computation resources will be consumed, leading to a higher Airbyte bill. - Set a higher `wal_keep_size`. If no unit is provided, it is in megabytes, and the default is `0`. See detailed documentation [here](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-KEEP-SIZE). The downside of this approach is that more disk space will be needed. @@ -404,6 +415,9 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.28 | 2022-11-22 | [19514](https://github.com/airbytehq/airbyte/pull/19514) | Adjust batch selection memory limits databases. | +| 1.0.27 | 2022-11-28 | [16990](https://github.com/airbytehq/airbyte/pull/16990) | Handle arrays data types | +| 1.0.26 | 2022-11-18 | [19551](https://github.com/airbytehq/airbyte/pull/19551) | Fixes bug with ssl modes | | 1.0.25 | 2022-11-16 | [19004](https://github.com/airbytehq/airbyte/pull/19004) | Use Debezium heartbeats to improve CDC replication of large databases. | | 1.0.24 | 2022-11-07 | [19291](https://github.com/airbytehq/airbyte/pull/19291) | Default timeout is reduced from 1 min to 10sec | | 1.0.23 | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025) | Stop enforce SSL if ssl mode is disabled | diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 37c92028567d..0132864049c9 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -22,7 +22,7 @@ This page guides you through the process of setting up the Stripe source connect We recommend creating a secret key specifically for Airbyte to control which resources Airbyte can access. For ease of use, we recommend granting read permission to all resources and configuring which resource to replicate in the Airbyte UI. You can also use the API keys for the [test mode](https://stripe.com/docs/keys#obtain-api-keys) to try out the Stripe integration with Airbyte. -7. For **Replication start date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. +7. For **Replication start date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. 8. For **Lookback Window in days (Optional)**, select the number of days the value in days prior to the start date that you to sync your data with. If your data is updated after setting up this connector, you can use the this option to reload data from the past N days. Example: If the Replication start date is set to `2021-01-01T00:00:00Z`, then: - If you leave the Lookback Window in days parameter to its the default value of 0, Airbyte will sync data from the Replication start date `2021-01-01T00:00:00Z` - If the Lookback Window in days value is set to 1, Airbyte will consider the Replication start date to be `2020-12-31T00:00:00Z` diff --git a/docs/integrations/sources/wrike.md b/docs/integrations/sources/wrike.md index 5674fbf38be2..0b2c8d49e35e 100644 --- a/docs/integrations/sources/wrike.md +++ b/docs/integrations/sources/wrike.md @@ -17,7 +17,7 @@ This page guides you through the process of setting up the Wrike source connecto Permissions granted to the permanent token are equal to the permissions of the user who generates the token. 6. For **Wrike Instance (hostname)**, add the hostname of the Wrike instance you are currently using. This could be `www.wrike.com`, `app-us2.wrike.com`, or anything similar. -7. For **Start date for comments**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The comments added on and after this date will be replicated. If this field is blank, Airbyte will replicate comments from the last seven days. +7. For **Start date for comments**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The comments added on and after this date will be replicated. If this field is blank, Airbyte will replicate comments from the last seven days. 8. Click **Set up source**. ## Supported sync modes diff --git a/docs/integrations/sources/xero.md b/docs/integrations/sources/xero.md index 1aeb70d9f3cf..1aa3a11ede06 100644 --- a/docs/integrations/sources/xero.md +++ b/docs/integrations/sources/xero.md @@ -45,7 +45,7 @@ As Xero uses .NET, some date fields in records could be in [.NET JSON date forma 6. For **Tenant ID** field, enter your Xero Organisation's [Tenant ID](https://developer.xero.com/documentation/guides/oauth2/auth-flow/#xero-tenants) 7. For **Scopes** field enter scopes you used for user's authorization on "Configuration" screen of your Xero App 8. Choose **Custom Connections Authentication** as **Authentication** option -9. For **Start date** enter UTC date and time in the format YYYY-MM-DDTHH:mm:ssZ as the start date and time of ingestion. +9. For **Start date** enter UTC date and time in the format `YYYY-MM-DDTHH:mm:ssZ` as the start date and time of ingestion. 10. Click **Set up source**. ## Supported sync modes @@ -55,4 +55,4 @@ The source connector supports the following [sync modes](https://docs.airbyte.co - Full Refresh - Incremental -## Changelog \ No newline at end of file +## Changelog diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index bbea4d6e52da..36ef03aa98fd 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -19,7 +19,7 @@ This page contains the setup guide and reference information for the Zendesk Cha 3. On the Set up the source page, select **Zendesk Chat** from the Source type dropdown. 4. Enter the name for the Zendesk Chat connector. 5. If you access Zendesk Chat from a [Zendesk subdomain](https://support.zendesk.com/hc/en-us/articles/4409381383578-Where-can-I-find-my-Zendesk-subdomain-), enter the **Subdomain**. -6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +6. For **Start Date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. 7. Click **Authenticate your Zendesk Chat account**. Log in and authorize your Zendesk Chat account. 8. Click **Set up source**. @@ -32,7 +32,7 @@ This page contains the setup guide and reference information for the Zendesk Cha 3. On the Set up the source page, select **Zendesk Chat** from the Source type dropdown. 4. Enter the name for the Zendesk Chat connector. 5. If you access Zendesk Chat from a [Zendesk subdomain](https://support.zendesk.com/hc/en-us/articles/4409381383578-Where-can-I-find-my-Zendesk-subdomain-), enter the **Subdomain**. -6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +6. For **Start Date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. 7. For Authorization Method, select **Access Token** from the dropdown and enter your Zendesk [access token](https://developer.zendesk.com/rest_api/docs/chat/auth). 8. Click **Set up source**. diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index a9d71998a51e..37fba3121fc5 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -14,7 +14,7 @@ This page guides you through setting up the Zendesk Support source connector. 3. On the Set up the source page, select **Zendesk Support** from the Source type dropdown. 4. Enter a name for your source. 5. For **Subdomain**, enter your [Zendesk subdomain](#prerequisites). -6. For **Start date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. +6. For **Start date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. 7. You can use OAuth or an API key to authenticate your Zendesk Support account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source. - To authenticate using OAuth for Airbyte Cloud, click **Authenticate your Zendesk Support account** to sign in with Zendesk Support and authorize your account. - To authenticate using an API key for Airbyte Open Source, select **API key** from the Authentication dropdown and enter your [API key](#prerequisites). Enter the **Email** associated with your Zendesk Support account. @@ -59,8 +59,9 @@ The Zendesk connector ideally should not run into Zendesk API limitations under ## Changelog | Version | Date | Pull Request | Subject | -|:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `0.2.17` | 2022-11-15 | [19432](https://github.com/airbytehq/airbyte/pull/19432) | Revert changes from version 0.2.15, use a test read instead | +|:---------|:-----------| :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0.2.18` | 2022-11-29 | [19432](https://github.com/airbytehq/airbyte/pull/19432) | Revert changes from version 0.2.15, use a test read instead | +| `0.2.17` | 2022-11-24 | [19792](https://github.com/airbytehq/airbyte/pull/19792) | Transform `ticket_comments.via` "-" to null | | `0.2.16` | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | | `0.2.15` | 2022-08-03 | [15233](https://github.com/airbytehq/airbyte/pull/15233) | Added `subscription plan` check on `streams discovery` step to remove streams that are not accessible for fetch due to subscription plan restrictions | | `0.2.14` | 2022-07-27 | [15036](https://github.com/airbytehq/airbyte/pull/15036) | Convert `ticket_audits.previous_value` values to string | diff --git a/docs/operator-guides/collecting-metrics.md b/docs/operator-guides/collecting-metrics.md index 6c21b274022b..03013efb75a6 100644 --- a/docs/operator-guides/collecting-metrics.md +++ b/docs/operator-guides/collecting-metrics.md @@ -266,3 +266,5 @@ Configure two additional env vars to the Datadog endpoint: ## Metrics Visit [OssMetricsRegistry.java](https://github.com/airbytehq/airbyte/blob/master/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java) to get a complete list of metrics Airbyte is sending. +## Additional information +Suppose you are looking for a non-production way of collecting metrics with dbt and Metabase, the tutorial [Airbyte Monitoring with dbt and Metabase](https://airbyte.com/blog/airbyte-monitoring-with-dbt-and-metabase) by accessing Airbyte's Postgres DB. The source code is open on [airbytehq/open-data-stack](https://github.com/airbytehq/open-data-stack). Think of it as an exploratory for data analysts and data engineers of building a dashboard on top of the existing Airbyte Postgres database versus the Prometheus more for DevOps engineers in production. diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 2d038c0e1a14..a897de8c7455 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -103,7 +103,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.40.21 --\ + docker run --rm -v /tmp:/config airbyte/migration:0.40.22 --\ --input /config/airbyte_archive.tar.gz\ --output /config/airbyte_archive_migrated.tar.gz ``` diff --git a/docs/project-overview/slack-code-of-conduct.md b/docs/project-overview/slack-code-of-conduct.md index cf38c7fe2a7f..c88da4c1adb5 100644 --- a/docs/project-overview/slack-code-of-conduct.md +++ b/docs/project-overview/slack-code-of-conduct.md @@ -13,7 +13,7 @@ All of the guidelines we provide below are important, but there’s a reason res ## Rule 2: Use the most relevant channels. -We deliberately use topic-specific Slack channels so members of the community can opt-in on various types of conversations. Our members take care to post their messages in the most relevant channel, and you’ll often see reminders about the best place to post a message (respectfully written, of course!). +We deliberately use topic-specific Slack channels so members of the community can opt-in on various types of conversations. Our members take care to post their messages in the most relevant channel, and you’ll often see reminders about the best place to post a message (respectfully written, of course!). If you're looking for help directly from the Community Assistance Team or other Airbyte employees, please stick to posting in the airbyte-help channel, so we know you're asking us specifically! ## Rule 3: Don’t double-post. diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 4b88a9f3fdff..24555ce4519d 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -12037,6 +12037,7 @@

WebBackendConnectionCreate
operations (optional)
array[OperationCreate]
sourceCatalogId (optional)
UUID format: uuid
geography (optional)
Geography
+
nonBreakingChangesPreference (optional)
NonBreakingChangesPreference
diff --git a/docs/troubleshooting/on-deploying.md b/docs/troubleshooting/on-deploying.md index e449752d4470..b51630bdebd8 100644 --- a/docs/troubleshooting/on-deploying.md +++ b/docs/troubleshooting/on-deploying.md @@ -95,3 +95,26 @@ ERROR: Head "https://registry-1.docker.io/v2/airbyte/init/manifests/{XXX}": unau You are most likely logged into Docker with your email address instead of your Docker ID. Log out of Docker by running `docker logout` and try running `docker-compose up` again. + +## Protocol Version errors from the bootloader when trying to upgrade + +When starting up Airbyte, the bootloader may fail with the following error: +``` +Aborting bootloader to avoid breaking existing connection after an upgrade. Please address airbyte protocol version support issues in the connectors before retrying. +``` + +We aborted the upgrade to avoid breaking existing connections due to a deprecation of protocol version. + +Looking at the `airbyte-bootloader` logs, there should be a few messages describing the change of support range of the Airbyte Protocol: + +``` +2022-11-21 22:07:20 INFO i.a.b.ProtocolVersionChecker(validate):81 - Detected an AirbyteProtocolVersion range change from [0.0.0:2.0.0] to [1.0.0:2.0.0] +2022-11-21 22:07:20 WARN i.a.b.ProtocolVersionChecker(validate):98 - The following connectors need to be upgraded before being able to upgrade the platform +2022-11-21 22:07:20 WARN j.u.s.ReferencePipeline$3$1(accept):197 - Source: d53f9084-fa6b-4a5a-976c-5b8392f4ad8a: E2E Testing: protocol version: 0.2.1 +``` + +From this example, this upgrade will drop the support for the major version 0 of the Airbyte Protocol. One connector here is problematic, we have the `E2E Testing` source connector that is blocking the upgrade because it is still using protocol version 0.2.1. + +In order to resolve this situation, all the problematic connectors must be upgraded to a version that is using a newer version of the Airbyte Protocol. In this specific example, we should target version 1 or 2, our recommendation is to always target the most recent version. + +For more details on how to upgrade a custom connector, you may want to refer to the [custom connector](../operator-guides/using-custom-connectors.md) documentation. diff --git a/docs/understanding-airbyte/airbyte-protocol-docker.md b/docs/understanding-airbyte/airbyte-protocol-docker.md index bfc3404715fd..8b630f2c7aac 100644 --- a/docs/understanding-airbyte/airbyte-protocol-docker.md +++ b/docs/understanding-airbyte/airbyte-protocol-docker.md @@ -43,7 +43,7 @@ cat <&0 | docker run --rm -i write --config ..` scheme for the Protocol Versioning. (see [SemVer](https://semver.org/)). + +We increment the +* MAJOR version when you make incompatible protocol changes +* MINOR version when you add functionality in a backwards compatible manner +* PATCH version when you make backwards compatible bug fixes + +## Development Guidelines + +1. We will continue to do our best effort to avoid introducing breaking changes to the Airbyte Protocol. +2. When introducing a new minor version of the Airbyte Protocol, new fields must come with sensible defaults for backward compatibility within the same major version, or be entirely optional. +3. When introducing a new major version of the Airbyte Protocol, all connectors from the previous major version will continue to work. This requires the ability to “translate” messages between 1 major version of the Airbyte Protocol. + +## Safeguards + +To ensure continuous operation, we have a few safeguards to prevent breaking existing configuration through protocol version incompatibilities. + +### When upgrading Airbyte + +When removing support for older versions of the Protocol, there is a risk removing the support for a version that is currently used. + +To mitigate this, as part of the pre-upgrade checks that happens in the `airbyte-bootloader`, we verify that any connector currently part of an active connection will still be supported after the upgrade. + +If any connector fails this check, we abort the upgrade and the `airbyte-bootloader` logs contains a list of connectors to upgrade. Those connectors will need to be upgraded from the UI before the platform itself can be upgraded. + +### When upgrading a Connector + +When upgrading a Connector from the UI, we will verify that the Protocol Version is supported before finalizing the Connector upgrade. \ No newline at end of file diff --git a/docs/understanding-airbyte/airbyte-protocol.md b/docs/understanding-airbyte/airbyte-protocol.md index d561b23ed07c..0df1118e86a8 100644 --- a/docs/understanding-airbyte/airbyte-protocol.md +++ b/docs/understanding-airbyte/airbyte-protocol.md @@ -190,6 +190,7 @@ The `connectionSpecification` is [JSONSchema](https://json-schema.org) that desc The specification also contains information about what features the Actor supports. +- `protocol_version` describes which version of the protocol the Connector supports. The default value is "0.2.0". - `supported_destination_sync_modes` - describes which sync modes a destination is able to support. See [Sync Modes](#source-sync-modes). `documentationUrl` and `changelogUrl` are optional fields that link to additional information about the connector. diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js index ff9c9630ab87..fad668bd3341 100644 --- a/docusaurus/sidebars.js +++ b/docusaurus/sidebars.js @@ -71,7 +71,7 @@ module.exports = { }, 'cloud/core-concepts', 'cloud/managing-airbyte-cloud', - + 'cloud/dbt-cloud-integration', ], }, { diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env index 2b3674e89f11..c0d8b1754bde 100644 --- a/kube/overlays/dev-integration-test/.env +++ b/kube/overlays/dev-integration-test/.env @@ -28,7 +28,9 @@ LOCAL_ROOT=/tmp/airbyte_local TRACKING_STRATEGY=logging WEBAPP_URL=airbyte-webapp-svc:80 API_URL=/api/v1/ +CONNECTOR_BUILDER_API_URL=/connector-builder-api INTERNAL_API_HOST=airbyte-server-svc:8001 +CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server-svc:8003 WORKER_ENVIRONMENT=kubernetes LOG_LEVEL=INFO @@ -90,3 +92,7 @@ ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS= WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true + +SHOULD_RUN_NOTIFY_WORKFLOWS=false +MAX_NOTIFY_WORKERS=5 +AUTO_DETECT_SCHEMA=false diff --git a/kube/overlays/dev-integration-test/kustomization.yaml b/kube/overlays/dev-integration-test/kustomization.yaml index 8344cb2ae51c..82d3e08b3742 100644 --- a/kube/overlays/dev-integration-test/kustomization.yaml +++ b/kube/overlays/dev-integration-test/kustomization.yaml @@ -21,8 +21,8 @@ images: newTag: 1.7.0 - name: airbyte/cron newTag: dev - #- name: airbyte/connector-builder-server #FIXME: Uncomment this block when enabling airbyte-connector-builder - # newTag: dev + - name: airbyte/connector-builder-server + newTag: dev configMapGenerator: - name: airbyte-env diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index 839afbaa47b1..8762e2ea5914 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -30,7 +30,9 @@ LOCAL_ROOT=/tmp/airbyte_local TRACKING_STRATEGY=logging WEBAPP_URL=airbyte-webapp-svc:80 API_URL=/api/v1/ +CONNECTOR_BUILDER_API_URL=/connector-builder-api INTERNAL_API_HOST=airbyte-server-svc:8001 +CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server-svc:8003 WORKER_ENVIRONMENT=kubernetes LOG_LEVEL=INFO @@ -92,3 +94,6 @@ ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS= WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true +SHOULD_RUN_NOTIFY_WORKFLOWS=false +MAX_NOTIFY_WORKERS=5 +AUTO_DETECT_SCHEMA=false diff --git a/kube/overlays/dev/kustomization.yaml b/kube/overlays/dev/kustomization.yaml index 100bf1be6b43..199d33d4f089 100644 --- a/kube/overlays/dev/kustomization.yaml +++ b/kube/overlays/dev/kustomization.yaml @@ -21,8 +21,8 @@ images: newTag: 1.7.0 - name: airbyte/cron newTag: dev - #- name: airbyte/connector-builder-server #FIXME: Uncomment this block when enabling airbyte-connector-builder - # newTag: dev + - name: airbyte/connector-builder-server + newTag: dev configMapGenerator: - name: airbyte-env diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 4f59e3c44860..f0f6896299df 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.40.21 +AIRBYTE_VERSION=0.40.22 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc @@ -30,7 +30,9 @@ LOCAL_ROOT=/tmp/airbyte_local TRACKING_STRATEGY=segment WEBAPP_URL=airbyte-webapp-svc:80 API_URL=/api/v1/ +CONNECTOR_BUILDER_API_URL=/connector-builder-api INTERNAL_API_HOST=airbyte-server-svc:8001 +CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server-svc:8003 WORKER_ENVIRONMENT=kubernetes LOG_LEVEL=INFO @@ -75,6 +77,7 @@ JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= # Launch a separate pod to orchestrate sync steps CONTAINER_ORCHESTRATOR_ENABLED=true +CONTAINER_ORCHESTRATOR_IMAGE= CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED=true @@ -93,3 +96,6 @@ ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS= WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true +SHOULD_RUN_NOTIFY_WORKFLOWS=false +MAX_NOTIFY_WORKERS=5 +AUTO_DETECT_SCHEMA=false diff --git a/kube/overlays/stable-with-resource-limits/kustomization.yaml b/kube/overlays/stable-with-resource-limits/kustomization.yaml index b9c038286671..ed86bc896933 100644 --- a/kube/overlays/stable-with-resource-limits/kustomization.yaml +++ b/kube/overlays/stable-with-resource-limits/kustomization.yaml @@ -8,21 +8,21 @@ bases: images: - name: airbyte/db - newTag: 0.40.21 + newTag: 0.40.22 - name: airbyte/bootloader - newTag: 0.40.21 + newTag: 0.40.22 - name: airbyte/server - newTag: 0.40.21 + newTag: 0.40.22 - name: airbyte/webapp - newTag: 0.40.21 + newTag: 0.40.22 - name: airbyte/worker - newTag: 0.40.21 + newTag: 0.40.22 - name: temporalio/auto-setup newTag: 1.7.0 - name: airbyte/cron - newTag: 0.40.21 - #- name: airbyte/connector-builder-server #FIXME: Uncomment this block when enabling airbyte-connector-builder - # newTag: 0.40.21 + newTag: 0.40.22 + - name: airbyte/connector-builder-server + newTag: 0.40.22 configMapGenerator: - name: airbyte-env diff --git a/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml b/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml index 283d7614b7f0..86d84d863021 100644 --- a/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml +++ b/kube/overlays/stable-with-resource-limits/set-resource-limits.yaml @@ -9,8 +9,8 @@ spec: - name: airbyte-db-container resources: limits: - cpu: 2 - memory: 2Gi + cpu: 0.5 + memory: 512Mi --- apiVersion: apps/v1 kind: Deployment @@ -29,8 +29,8 @@ spec: key: CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED resources: limits: - cpu: 2 - memory: 1Gi + cpu: 0.5 + memory: 512Mi --- apiVersion: apps/v1 kind: Deployment @@ -49,8 +49,8 @@ spec: key: CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED resources: limits: - cpu: 1 - memory: 2Gi + cpu: 0.5 + memory: 512Mi --- apiVersion: apps/v1 kind: Deployment @@ -63,7 +63,7 @@ spec: - name: airbyte-temporal resources: limits: - cpu: 2 + cpu: 0.5 memory: 512Mi --- apiVersion: apps/v1 @@ -77,7 +77,7 @@ spec: - name: airbyte-webapp-container resources: limits: - cpu: 1 + cpu: 0.5 memory: 512Mi --- apiVersion: apps/v1 @@ -91,7 +91,7 @@ spec: - name: airbyte-minio resources: limits: - cpu: 1 + cpu: 0.5 memory: 512Mi --- apiVersion: apps/v1 @@ -121,17 +121,17 @@ spec: limits: cpu: 0.5 memory: 128Mi -#--- #FIXME: Uncomment this block when enabling airbyte-connector-builder -#apiVersion: apps/v1 -#kind: Deployment -#metadata: -# name: airbyte-connector-builder-server -#spec: -# template: -# spec: -# containers: -# - name: airbyte-connector-builder-server-container -# resources: -# limits: -# cpu: 0.5 -# memory: 128Mi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: airbyte-connector-builder-server +spec: + template: + spec: + containers: + - name: airbyte-connector-builder-server-container + resources: + limits: + cpu: 0.5 + memory: 128Mi diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 213ab11e0c35..25f02040322e 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.40.21 +AIRBYTE_VERSION=0.40.22 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc @@ -30,7 +30,9 @@ LOCAL_ROOT=/tmp/airbyte_local TRACKING_STRATEGY=segment WEBAPP_URL=airbyte-webapp-svc:80 API_URL=/api/v1/ +CONNECTOR_BUILDER_API_URL=/connector-builder-api INTERNAL_API_HOST=airbyte-server-svc:8001 +CONNECTOR_BUILDER_API_HOST=airbyte-connector-builder-server-svc:8003 WORKER_ENVIRONMENT=kubernetes LOG_LEVEL=INFO @@ -92,3 +94,6 @@ ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS= WORKFLOW_FAILURE_RESTART_DELAY_SECONDS= USE_STREAM_CAPABLE_STATE=true +SHOULD_RUN_NOTIFY_WORKFLOWS=false +MAX_NOTIFY_WORKERS=5 +AUTO_DETECT_SCHEMA=false diff --git a/kube/overlays/stable/kustomization.yaml b/kube/overlays/stable/kustomization.yaml index f6a3f08c9e4f..d1ee6b567fde 100644 --- a/kube/overlays/stable/kustomization.yaml +++ b/kube/overlays/stable/kustomization.yaml @@ -8,21 +8,21 @@ bases: images: - name: airbyte/db - newTag: 0.40.21 + newTag: 0.40.22 - name: airbyte/bootloader - newTag: 0.40.21 + newTag: 0.40.22 - name: airbyte/server - newTag: 0.40.21 + newTag: 0.40.22 - name: airbyte/webapp - newTag: 0.40.21 + newTag: 0.40.22 - name: airbyte/worker - newTag: 0.40.21 + newTag: 0.40.22 - name: temporalio/auto-setup newTag: 1.7.0 - name: airbyte/cron - newTag: 0.40.21 - #- name: airbyte/connector-builder-server #FIXME: Uncomment this block when enabling airbyte-connector-builder - # newTag: 0.40.21 + newTag: 0.40.22 + - name: airbyte/connector-builder-server + newTag: 0.40.22 configMapGenerator: - name: airbyte-env diff --git a/kube/resources/connector-builder-server.yaml b/kube/resources/connector-builder-server.yaml index bd6f5f98d625..08dfc737d080 100644 --- a/kube/resources/connector-builder-server.yaml +++ b/kube/resources/connector-builder-server.yaml @@ -1,3 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: airbyte-connector-builder-server-svc +spec: + type: NodePort + ports: + - port: 8003 + protocol: TCP + targetPort: 80 + selector: + airbyte: connector-builder-server +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -21,3 +34,5 @@ spec: configMapKeyRef: name: airbyte-env key: AIRBYTE_VERSION + ports: + - containerPort: 80 diff --git a/kube/resources/kustomization.yaml b/kube/resources/kustomization.yaml index be3d0af1859f..bb4cd4dd3793 100644 --- a/kube/resources/kustomization.yaml +++ b/kube/resources/kustomization.yaml @@ -4,7 +4,7 @@ kind: Kustomization resources: - airbyte-minio.yaml - bootloader.yaml - #- connector-builder-server.yaml #FIXME uncomment this when enabling the connector-builder + - connector-builder-server.yaml - cron.yaml - db.yaml - pod-sweeper.yaml diff --git a/kube/resources/server.yaml b/kube/resources/server.yaml index 02a7c3b87550..2892560a6345 100644 --- a/kube/resources/server.yaml +++ b/kube/resources/server.yaml @@ -153,6 +153,11 @@ spec: configMapKeyRef: name: airbyte-env key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: AUTO_DETECT_SCHEMA + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AUTO_DETECT_SCHEMA ports: - containerPort: 8001 volumeMounts: diff --git a/kube/resources/webapp.yaml b/kube/resources/webapp.yaml index 9bb5c59ac68a..f057961526bb 100644 --- a/kube/resources/webapp.yaml +++ b/kube/resources/webapp.yaml @@ -38,6 +38,11 @@ spec: configMapKeyRef: name: airbyte-env key: API_URL + - name: CONNECTOR_BUILDER_API_URL + valueFrom: + configMapKeyRef: + name: airbyte-env + key: CONNECTOR_BUILDER_API_URL - name: TRACKING_STRATEGY valueFrom: configMapKeyRef: @@ -48,5 +53,10 @@ spec: configMapKeyRef: name: airbyte-env key: INTERNAL_API_HOST + - name: CONNECTOR_BUILDER_API_HOST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: CONNECTOR_BUILDER_API_HOST ports: - containerPort: 80 diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index 75cba0fd2c49..d8f7be704011 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -270,6 +270,11 @@ spec: configMapKeyRef: name: airbyte-env key: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + - name: AUTO_DETECT_SCHEMA + valueFrom: + configMapKeyRef: + name: airbyte-env + key: AUTO_DETECT_SCHEMA - name: USE_STREAM_CAPABLE_STATE valueFrom: configMapKeyRef: diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile index c1ff8705d946..1c9e2c7883a3 100644 --- a/octavia-cli/Dockerfile +++ b/octavia-cli/Dockerfile @@ -14,5 +14,5 @@ USER octavia-cli WORKDIR /home/octavia-project ENTRYPOINT ["octavia"] -LABEL io.airbyte.version=0.40.21 +LABEL io.airbyte.version=0.40.22 LABEL io.airbyte.name=airbyte/octavia-cli diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 0e788a7a2224..28fe67f5f888 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -104,7 +104,7 @@ This script: ```bash touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.40.21 +docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.40.22 ``` ### Using `docker-compose` @@ -712,7 +712,7 @@ You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment | Version | Date | Description | PR | | ------- | ---------- | ------------------------------------------------------------------------------------- | ----------------------------------------------------------- | | 0.41.0 | 2022-10-13 | Use Basic Authentication for making API requests | [#17982](https://github.com/airbytehq/airbyte/pull/17982) | -| 0.40.21 | 2022-08-10 | Enable cron and basic scheduling | [#15253](https://github.com/airbytehq/airbyte/pull/15253) | +| 0.40.22 | 2022-08-10 | Enable cron and basic scheduling | [#15253](https://github.com/airbytehq/airbyte/pull/15253) | | 0.39.33 | 2022-07-05 | Add `octavia import all` command | [#14374](https://github.com/airbytehq/airbyte/pull/14374) | | 0.39.32 | 2022-06-30 | Create import command to import and manage existing Airbyte resource from octavia-cli | [#14137](https://github.com/airbytehq/airbyte/pull/14137) | | 0.39.27 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) | diff --git a/octavia-cli/install.sh b/octavia-cli/install.sh index 62a4d20a3473..1f546a882e4c 100755 --- a/octavia-cli/install.sh +++ b/octavia-cli/install.sh @@ -3,7 +3,7 @@ # This install scripts currently only works for ZSH and Bash profiles. # It creates an octavia alias in your profile bound to a docker run command and your current user. -VERSION=0.40.21 +VERSION=0.40.22 OCTAVIA_ENV_FILE=${HOME}/.octavia detect_profile() { diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index bd572194b14b..8ee47ac56bae 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -15,7 +15,7 @@ setup( name="octavia-cli", - version="0.40.21", + version="0.40.22", description="A command line interface to manage Airbyte configurations", long_description=README, author="Airbyte", diff --git a/pyproject.toml b/pyproject.toml index 437bc1a8d0a4..17544d0afc31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ color_output = true error_summary = true [tool.pytest.ini_options] -minversion = "6.0" +minversion = "6.2.5" addopts ="-r a --capture=no -vv --log-level=DEBUG --color=yes" diff --git a/tools/openapi2jsonschema/examples/airbyte.local/openapi.yaml b/tools/openapi2jsonschema/examples/airbyte.local/openapi.yaml index 30c32da65c21..84c985df000f 100644 --- a/tools/openapi2jsonschema/examples/airbyte.local/openapi.yaml +++ b/tools/openapi2jsonschema/examples/airbyte.local/openapi.yaml @@ -2638,6 +2638,11 @@ components: $ref: "#/components/schemas/ConnectionStatus" syncCatalog: $ref: "#/components/schemas/AirbyteCatalog" + nonBreakingChangesPreference: + enum: + - ignore + - disable + type: string required: - connection - sourceId