Skip to content

Commit

Permalink
Merge branch 'master' into leti/tools-add-posthog
Browse files Browse the repository at this point in the history
* master: (304 commits)
  Bump helm chart version reference to 0.40.27 (#18152)
  Bump helm chart version reference to 0.40.26 (#18094)
  Update deployment.yaml (#18151)
  Publishes Postgres, MySQL, MSSQL source with changes from #18041 (#18086)
  Fix minor DBT Cloud Errors. (#18147)
  Sentry Integration : Stop reporting all non system-error error types. (#18133)
  Docs: Fix backoff stategy docs (#18143)
  🐛 Destination GCS: Fix error logs to log 'Gcs' rather than 'AWS' (#17901)
  Add openAPI spec for Connector Builder Server (#17535)
  Alex/mvp UI for dbt cloud integration (#18095)
  increased timeout for sat tests (#18128)
  Bmoric/remove dep connector worker (#17977)
  `recordsRead` should be a long (#18123)
  doc_update_oath_issue_gsc (#17967)
  🎉 Source Zendesk Chat: engagements data fix infinity looping + gradlew format (#18121)
  🐛 Source Zendesk Chat: engagements data fix infinity looping (#17745)
  Custom APM Tracing (#17947)
  11679 BigQuery-Denormalized Destination: improve code coverage (#17827)
  increased timeout for sat tests (#18114)
  docs: clarify language (#18090)
  ...
  • Loading branch information
letiescanciano committed Oct 19, 2022
2 parents da478e6 + 8b68308 commit e3bb500
Show file tree
Hide file tree
Showing 1,719 changed files with 38,763 additions and 23,680 deletions.
4 changes: 1 addition & 3 deletions .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.40.10
current_version = 0.40.15
commit = False
tag = False
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
Expand Down Expand Up @@ -38,8 +38,6 @@ serialize =

[bumpversion:file:charts/airbyte-bootloader/Chart.yaml]

[bumpversion:file:charts/airbyte/values.yaml]

[bumpversion:file:charts/airbyte/README.md]

[bumpversion:file:docs/operator-guides/upgrading-airbyte.md]
Expand Down
6 changes: 4 additions & 2 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@


### SHARED ###
VERSION=0.40.10
VERSION=0.40.15

# When using the airbyte-db via default docker image
CONFIG_ROOT=/data
Expand Down Expand Up @@ -81,6 +81,8 @@ LOG_LEVEL=INFO
### APPLICATIONS ###
# Worker #
WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
# Cron #
CRON_MICRONAUT_ENVIRONMENTS=control-plane
# Relevant to scaling.
MAX_SYNC_WORKERS=5
MAX_SPEC_WORKERS=5
Expand All @@ -102,4 +104,4 @@ METRIC_CLIENT=
# Useful only when metric client is set to be otel. Must start with http:// or https://.
OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"

USE_STREAM_CAPABLE_STATE=true
USE_STREAM_CAPABLE_STATE=true
1 change: 1 addition & 0 deletions .env.dev
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ INTERNAL_API_HOST=airbyte-server:8001
SYNC_JOB_MAX_ATTEMPTS=3
SYNC_JOB_MAX_TIMEOUT_DAYS=3
WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
CRON_MICRONAUT_ENVIRONMENTS=control-plane

# Sentry
SENTRY_DSN=""
Expand Down
142 changes: 142 additions & 0 deletions .github/workflows/commands-for-testing-tool.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
name: Run Testing Tool Commands
on:
issue_comment:
types: [created]
jobs:
set-params:
# Only allow slash commands on pull request (not on issues)
if: ${{ github.event.issue.pull_request }}
runs-on: ubuntu-latest
outputs:
repo: ${{ steps.getref.outputs.repo }}
ref: ${{ steps.getref.outputs.ref }}
comment-id: ${{ steps.comment-info.outputs.comment-id }}
command: ${{ steps.regex.outputs.first_match }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Check PAT rate limits
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.AIRBYTEIO_PAT }} \
${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
${{ secrets.SUPERTOPHER_PAT }} \
${{ secrets.DAVINCHIA_PAT }}
- name: Get PR repo and ref
id: getref
run: |
pr_info="$(curl ${{ github.event.issue.pull_request.url }})"
echo ::set-output name=ref::"$(echo $pr_info | jq -r '.head.ref')"
echo ::set-output name=repo::"$(echo $pr_info | jq -r '.head.repo.full_name')"
- name: Get comment id
id: comment-info
run: |
echo ::set-output name=comment-id::"${{ github.event.comment.id }}"
- name: Get command
id: regex
uses: AsasInnab/regex-action@v1
with:
regex_pattern: "^/[a-zA-Z0-9_/-]+"
regex_flags: "i"
search_string: ${{ github.event.comment.body }}
helps-run:
runs-on: ubuntu-latest
if: |
needs.set-params.outputs.command == '/help-full' ||
needs.set-params.outputs.command == '/help' ||
needs.set-params.outputs.command == '/list-scenarios'
needs: set-params
steps:
- name: Update comment for processing
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
reactions: eyes, rocket
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Pull Testing Tool docker image
run: docker pull airbyte/airbyte-e2e-testing-tool:latest
- name: Create input and output folders
run: |
mkdir secrets
mkdir result
- name: Run docker container with params
run: docker run -v $(pwd)/secrets:/secrets -v $(pwd)/result:/result airbyte/airbyte-e2e-testing-tool:latest ${{ github.event.comment.body }}
- name: Read file with results
id: read_file
uses: andstor/file-reader-action@v1
with:
path: "result/log"
- name: Add Success Comment
if: needs.set-params.outputs.comment-id && success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
body: |
> :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
${{ steps.read_file.outputs.contents }}
reactions: +1
- name: Add Failure Comment
if: needs.set-params.outputs.comment-id && failure()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
body: |
> :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
reactions: -1
scenarios-run:
runs-on: ubuntu-latest
if: |
needs.set-params.outputs.command == '/run-scenario' ||
needs.set-params.outputs.command == '/run-scenario-local'
needs: set-params
steps:
- name: Update comment for processing
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
reactions: eyes, rocket
- name: Checkout Airbyte
uses: actions/checkout@v2
with:
repository: ${{ needs.set-params.outputs.repo }}
ref: ${{ needs.set-params.outputs.ref }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Pull Testing Tool docker image
run: docker pull airbyte/airbyte-e2e-testing-tool:latest
- name: Change wrapper permissions
run: |
mkdir secrets
mkdir result
- name: Run Airbyte
run: docker-compose up -d
- name: Connect to secret manager
uses: jsdaniell/create-json@1.1.2
with:
name: "/secrets/service_account_credentials.json"
json: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }}
- name: Run docker container with params
run: docker run -v $(pwd)/secrets:/secrets -v $(pwd)/result:/result airbyte/airbyte-e2e-testing-tool:latest ${{ github.event.comment.body }}
- name: Read file with results
id: read_file
uses: andstor/file-reader-action@v1
with:
path: "result/log"
- name: Add Success Comment
if: needs.set-params.outputs.comment-id && success()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
body: |
> :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
${{ steps.read_file.outputs.contents }}
reactions: +1
- name: Add Failure Comment
if: needs.set-params.outputs.comment-id && failure()
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ needs.set-params.outputs.comment-id }}
body: |
> :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
reactions: -1
43 changes: 42 additions & 1 deletion .github/workflows/gradle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ jobs:
filters: |
backend:
- 'airbyte-!(cdk|integrations|webapp|webapp-e2e-tests)/**'
- 'airbyte-integrations/connectors/(destination-jdbc|destination-postgres|source-jdbc|source-postgres)/**'
- 'airbyte-config/init/src/main/resources/seed/(source|destination)_definitions.yaml'
build:
- '.github/**'
- 'buildSrc/**'
Expand All @@ -59,6 +61,7 @@ jobs:
- 'airbyte-cdk/**'
- 'airbyte-protocol/**'
- 'airbyte-integrations/**'
- 'airbyte-commons-worker/**'
db:
- 'airbyte-db/**'
frontend:
Expand Down Expand Up @@ -229,7 +232,7 @@ jobs:
run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace

- name: Build
run: SUB_BUILD=CONNECTORS_BASE ./gradlew build -x :airbyte-workers:test --scan
run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan

- name: Ensure no file change
run: git --no-pager diff && test -z "$(git --no-pager diff)"
Expand Down Expand Up @@ -530,6 +533,25 @@ jobs:
path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml'
reporter: java-junit

- name: Upload test results to BuildPulse for flaky test detection
if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled.
uses: Workshop64/buildpulse-action@main
with:
account: 59758427
repository: 283046497
path: '/actions-runner/_work/airbyte/airbyte/*'
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}

- name: Upload test results to Github for analysis
if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled.
uses: actions/upload-artifact@v3
with:
path: |
/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml
/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml
name: test-results-build

# In case of self-hosted EC2 errors, remove this block.
stop-platform-build-runner:
name: "Platform: Stop Build EC2 Runner"
Expand Down Expand Up @@ -669,6 +691,25 @@ jobs:
path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
reporter: java-junit

- name: Upload test results to BuildPulse for flaky test detection
if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled.
uses: Workshop64/buildpulse-action@main
with:
account: 59758427
repository: 283046497
path: '/actions-runner/_work/airbyte/airbyte/*'
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}

- name: Upload test results to Github for analysis
if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled.
uses: actions/upload-artifact@v3
with:
path: |
/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml
/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml
name: test-results-kube

- uses: actions/upload-artifact@v2
if: failure()
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/publish-command.yml
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ jobs:
if: github.event.inputs.auto-bump-version == 'true' && success()
run: |
git add -u
git commit -m "auto-bump connector version [ci skip]"
git commit -m "auto-bump connector version"
git pull origin ${{ github.event.inputs.gitref }}
git push origin ${{ github.event.inputs.gitref }}
id: auto-bump
Expand Down
51 changes: 49 additions & 2 deletions .github/workflows/publish-helm-charts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,18 @@ on:
- 'master'
paths:
- 'charts/**'
- '!charts/**/Chart.yaml'
- '!charts/**/Chart.lock'
- '!charts/**/README.md'
workflow_dispatch:

jobs:
generate-semantic-version:
name: Generate semantic version
runs-on: ubuntu-22.04
outputs:
next-version: ${{ steps.sem-ver.outputs.version }}
next-version: ${{ steps.ver.outputs.fragment }}
tag: ${{ steps.sem-ver.outputs.version_tag }}
steps:
- uses: actions/checkout@v2
with:
Expand All @@ -30,11 +34,21 @@ jobs:
format: "${major}.${minor}.${patch}"
change_path: "./charts"
bump_each_commit: true
namespace: helm

- name: "Remove -helm from ver number"
shell: bash
env:
VERSION: ${{ steps.sem-ver.outputs.version }}
id: ver
run: echo "::set-output name=fragment::${VERSION%%-*}"


release-chart:
name: Chart release
runs-on: ubuntu-22.04
needs: ["generate-semantic-version"]
permissions: write-all
steps:
- uses: actions/checkout@v3
with:
Expand All @@ -51,14 +65,17 @@ jobs:
shell: bash
working-directory: ./airbyte/charts
run: |
sed -i "s/ version: placeholder/ version: ${{ needs.generate-semantic-version.outputs.next-version }}/g" airbyte/Chart.yaml
sed -i -E "s/ version: [[:digit:]].[[:digit:]].[[:digit:]]/ version: ${{ needs.generate-semantic-version.outputs.next-version }}/g" airbyte/Chart.yaml
sed -i -E 's/version: [0-9]+\.[0-9]+\.[0-9]+/version: ${{ needs.generate-semantic-version.outputs.next-version }}/' airbyte/Chart.yaml
- name: "Helm package"
shell: bash
run: |
declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics")
for val in ${StringArray[@]}; do
cd ./airbyte/charts/${val} && helm dep update && cd $GITHUB_WORKSPACE
sed -i -E 's/version: \"[0-9]+\.[0-9]+\.[0-9]+\"/version: \"${{ needs.generate-semantic-version.outputs.next-version }}\"/' ./airbyte/charts/${val}/Chart.yaml
helm package ./airbyte/charts/${val} -d airbyte-oss --version ${{ needs.generate-semantic-version.outputs.next-version }}
done
helm repo index airbyte-oss/
Expand Down Expand Up @@ -89,3 +106,33 @@ jobs:
add: '.'
cwd: './airbyte-oss/'


- name: "Generate changelog"
shell: bash
id: changelog
run: |
cd ./airbyte/
echo "::set-output name=changelog::$(PAGER=cat git log $(git describe --tags --match "*-helm" $(git rev-list --tags --max-count=1))..HEAD --oneline --decorate=no)"
- name: Create Pull Request
uses: peter-evans/create-pull-request@v4
with:
path: ./airbyte/
branch: update-helm-chart-version-ref
branch-suffix: random
title: Bump helm chart version reference to ${{ needs.generate-semantic-version.outputs.next-version }}
body: |
## What
Bump version reference in all Chart.yaml files to ${{ needs.generate-semantic-version.outputs.next-version }}
CHANGELOG:
${{ steps.changelog.outputs.changelog }}
commit-message: Bump helm chart version reference to ${{ needs.generate-semantic-version.outputs.next-version }}
delete-branch: true

- name: Create tag
shell: bash
run: |
cd ./airbyte/
git tag ${{ needs.generate-semantic-version.outputs.tag }}
git push origin ${{ needs.generate-semantic-version.outputs.tag }}
Loading

0 comments on commit e3bb500

Please sign in to comment.