[AP-1772] Added size check option for sync_tables command #623
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End Tests | |
| on: | |
| pull_request: | |
| branches: [master] | |
| workflow_dispatch: | |
| concurrency: | |
| group: e2e_tests-${{ github.head_ref }} | |
| cancel-in-progress: true | |
| env: | |
| TARGET_SNOWFLAKE_ACCOUNT: ${{ secrets.TARGET_SNOWFLAKE_ACCOUNT }} | |
| TARGET_SNOWFLAKE_AWS_ACCESS_KEY: ${{ secrets.TARGET_SNOWFLAKE_AWS_ACCESS_KEY }} | |
| TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY: ${{ secrets.TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY }} | |
| TARGET_SNOWFLAKE_DBNAME: ${{ secrets.TARGET_SNOWFLAKE_DBNAME }} | |
| TARGET_SNOWFLAKE_FILE_FORMAT: ${{ secrets.TARGET_SNOWFLAKE_FILE_FORMAT }} | |
| TARGET_SNOWFLAKE_PASSWORD: ${{ secrets.TARGET_SNOWFLAKE_PASSWORD }} | |
| TARGET_SNOWFLAKE_S3_BUCKET: ${{ secrets.TARGET_SNOWFLAKE_S3_BUCKET }} | |
| TARGET_SNOWFLAKE_S3_KEY_PREFIX: ${{ secrets.TARGET_SNOWFLAKE_S3_KEY_PREFIX }} | |
| TARGET_SNOWFLAKE_SCHEMA: ${{ secrets.TARGET_SNOWFLAKE_SCHEMA }} | |
| TARGET_SNOWFLAKE_STAGE: ${{ secrets.TARGET_SNOWFLAKE_STAGE }} | |
| TARGET_SNOWFLAKE_USER: ${{ secrets.TARGET_SNOWFLAKE_USER }} | |
| TARGET_SNOWFLAKE_WAREHOUSE: ${{ secrets.TARGET_SNOWFLAKE_WAREHOUSE }} | |
| TAP_S3_CSV_AWS_KEY: ${{ secrets.TAP_S3_CSV_AWS_KEY }} | |
| TAP_S3_CSV_AWS_SECRET_ACCESS_KEY: ${{ secrets.TAP_S3_CSV_AWS_SECRET_ACCESS_KEY }} | |
| TAP_S3_CSV_BUCKET: ${{ secrets.TAP_S3_CSV_BUCKET }} | |
| jobs: | |
| e2e_tests_target_pg: | |
| runs-on: ubuntu-20.04 | |
| environment: ci_tests | |
| steps: | |
| - name: Checking out repo | |
| uses: actions/checkout@v4.1.7 | |
| - name: Check if python changes are present | |
| id: check | |
| env: | |
| GITHUB_REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| continue-on-error: true | |
| run: ./scripts/ci_check_no_file_changes.sh python | |
| - name: Setup test containers | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| cp dev-project/.env.template dev-project/.env | |
| docker compose -f dev-project/docker-compose.yml up -d | |
| - name: Wait for test containers to be ready | |
| timeout-minutes: 5 | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| until docker logs pipelinewise_dev | grep "PipelineWise Dev environment is ready" | |
| do | |
| echo 'Sleeping for 10s'; | |
| sleep 10; | |
| done | |
| - name: Run target postgres end-to-end tests | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| docker exec -t \ | |
| -e TAP_S3_CSV_AWS_KEY=$TAP_S3_CSV_AWS_KEY \ | |
| -e TAP_S3_CSV_AWS_SECRET_ACCESS_KEY=$TAP_S3_CSV_AWS_SECRET_ACCESS_KEY \ | |
| -e TAP_S3_CSV_BUCKET=$TAP_S3_CSV_BUCKET \ | |
| pipelinewise_dev pytest tests/end_to_end/test_target_postgres.py -vx --timer-top-n 10 | |
| e2e_tests_mariadb_to_sf: | |
| runs-on: ubuntu-20.04 | |
| environment: ci_tests | |
| steps: | |
| - name: Checking out repo | |
| uses: actions/checkout@v4.1.7 | |
| - name: Check if python changes are present | |
| id: check | |
| env: | |
| GITHUB_REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| continue-on-error: true | |
| run: ./scripts/ci_check_no_file_changes.sh python | |
| - name: Setup test containers | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| cp dev-project/.env.template dev-project/.env | |
| docker compose -f dev-project/docker-compose.yml up -d | |
| - name: Wait for test containers to be ready | |
| if: steps.check.outcome == 'failure' | |
| timeout-minutes: 5 | |
| run: | | |
| until docker logs pipelinewise_dev | grep "PipelineWise Dev environment is ready" | |
| do | |
| echo 'Sleeping for 10s'; | |
| sleep 10; | |
| done | |
| - name: Run target snowflake end-to-end tests | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| docker exec -t \ | |
| -e TARGET_SNOWFLAKE_ACCOUNT=$TARGET_SNOWFLAKE_ACCOUNT \ | |
| -e TARGET_SNOWFLAKE_AWS_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_ACCESS_KEY \ | |
| -e TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY \ | |
| -e TARGET_SNOWFLAKE_DBNAME=$TARGET_SNOWFLAKE_DBNAME \ | |
| -e TARGET_SNOWFLAKE_FILE_FORMAT=$TARGET_SNOWFLAKE_FILE_FORMAT \ | |
| -e TARGET_SNOWFLAKE_PASSWORD=$TARGET_SNOWFLAKE_PASSWORD \ | |
| -e TARGET_SNOWFLAKE_S3_BUCKET=$TARGET_SNOWFLAKE_S3_BUCKET \ | |
| -e TARGET_SNOWFLAKE_S3_KEY_PREFIX=$TARGET_SNOWFLAKE_S3_KEY_PREFIX \ | |
| -e TARGET_SNOWFLAKE_SCHEMA=$TARGET_SNOWFLAKE_SCHEMA \ | |
| -e TARGET_SNOWFLAKE_STAGE=$TARGET_SNOWFLAKE_STAGE \ | |
| -e TARGET_SNOWFLAKE_USER=$TARGET_SNOWFLAKE_USER \ | |
| -e TARGET_SNOWFLAKE_WAREHOUSE=$TARGET_SNOWFLAKE_WAREHOUSE \ | |
| pipelinewise_dev pytest tests/end_to_end/target_snowflake/tap_mariadb -vx --timer-top-n 10 | |
| e2e_tests_pg_to_sf: | |
| runs-on: ubuntu-20.04 | |
| environment: ci_tests | |
| steps: | |
| - name: Checking out repo | |
| uses: actions/checkout@v4.1.7 | |
| - name: Check if python changes are present | |
| id: check | |
| env: | |
| GITHUB_REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| continue-on-error: true | |
| run: ./scripts/ci_check_no_file_changes.sh python | |
| - name: Setup test containers | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| cp dev-project/.env.template dev-project/.env | |
| docker compose -f dev-project/docker-compose.yml up -d | |
| - name: Wait for test containers to be ready | |
| if: steps.check.outcome == 'failure' | |
| timeout-minutes: 5 | |
| run: | | |
| until docker logs pipelinewise_dev | grep "PipelineWise Dev environment is ready" | |
| do | |
| echo 'Sleeping for 10s'; | |
| sleep 10; | |
| done | |
| - name: Run target snowflake end-to-end tests | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| docker exec -t \ | |
| -e TARGET_SNOWFLAKE_ACCOUNT=$TARGET_SNOWFLAKE_ACCOUNT \ | |
| -e TARGET_SNOWFLAKE_AWS_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_ACCESS_KEY \ | |
| -e TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY \ | |
| -e TARGET_SNOWFLAKE_DBNAME=$TARGET_SNOWFLAKE_DBNAME \ | |
| -e TARGET_SNOWFLAKE_FILE_FORMAT=$TARGET_SNOWFLAKE_FILE_FORMAT \ | |
| -e TARGET_SNOWFLAKE_PASSWORD=$TARGET_SNOWFLAKE_PASSWORD \ | |
| -e TARGET_SNOWFLAKE_S3_BUCKET=$TARGET_SNOWFLAKE_S3_BUCKET \ | |
| -e TARGET_SNOWFLAKE_S3_KEY_PREFIX=$TARGET_SNOWFLAKE_S3_KEY_PREFIX \ | |
| -e TARGET_SNOWFLAKE_SCHEMA=$TARGET_SNOWFLAKE_SCHEMA \ | |
| -e TARGET_SNOWFLAKE_STAGE=$TARGET_SNOWFLAKE_STAGE \ | |
| -e TARGET_SNOWFLAKE_USER=$TARGET_SNOWFLAKE_USER \ | |
| -e TARGET_SNOWFLAKE_WAREHOUSE=$TARGET_SNOWFLAKE_WAREHOUSE \ | |
| pipelinewise_dev pytest tests/end_to_end/target_snowflake/tap_postgres -vx --timer-top-n 10 | |
| e2e_tests_mg_to_sf: | |
| runs-on: ubuntu-20.04 | |
| environment: ci_tests | |
| steps: | |
| - name: Checking out repo | |
| uses: actions/checkout@v4.1.7 | |
| - name: Check if python changes are present | |
| id: check | |
| env: | |
| GITHUB_REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| continue-on-error: true | |
| run: ./scripts/ci_check_no_file_changes.sh python | |
| - name: Setup test containers | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| cp dev-project/.env.template dev-project/.env | |
| docker compose -f dev-project/docker-compose.yml up -d | |
| - name: Wait for test containers to be ready | |
| if: steps.check.outcome == 'failure' | |
| timeout-minutes: 5 | |
| run: | | |
| until docker logs pipelinewise_dev | grep "PipelineWise Dev environment is ready" | |
| do | |
| echo 'Sleeping for 10s'; | |
| sleep 10; | |
| done | |
| - name: Run target snowflake end-to-end tests | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| docker exec -t \ | |
| -e TARGET_SNOWFLAKE_ACCOUNT=$TARGET_SNOWFLAKE_ACCOUNT \ | |
| -e TARGET_SNOWFLAKE_AWS_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_ACCESS_KEY \ | |
| -e TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY \ | |
| -e TARGET_SNOWFLAKE_DBNAME=$TARGET_SNOWFLAKE_DBNAME \ | |
| -e TARGET_SNOWFLAKE_FILE_FORMAT=$TARGET_SNOWFLAKE_FILE_FORMAT \ | |
| -e TARGET_SNOWFLAKE_PASSWORD=$TARGET_SNOWFLAKE_PASSWORD \ | |
| -e TARGET_SNOWFLAKE_S3_BUCKET=$TARGET_SNOWFLAKE_S3_BUCKET \ | |
| -e TARGET_SNOWFLAKE_S3_KEY_PREFIX=$TARGET_SNOWFLAKE_S3_KEY_PREFIX \ | |
| -e TARGET_SNOWFLAKE_SCHEMA=$TARGET_SNOWFLAKE_SCHEMA \ | |
| -e TARGET_SNOWFLAKE_STAGE=$TARGET_SNOWFLAKE_STAGE \ | |
| -e TARGET_SNOWFLAKE_USER=$TARGET_SNOWFLAKE_USER \ | |
| -e TARGET_SNOWFLAKE_WAREHOUSE=$TARGET_SNOWFLAKE_WAREHOUSE \ | |
| pipelinewise_dev pytest tests/end_to_end/target_snowflake/tap_mongodb -vx --timer-top-n 10 | |
| e2e_tests_s3_to_sf: | |
| runs-on: ubuntu-20.04 | |
| environment: ci_tests | |
| steps: | |
| - name: Checking out repo | |
| uses: actions/checkout@v4.1.7 | |
| - name: Check if python changes are present | |
| id: check | |
| env: | |
| GITHUB_REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| continue-on-error: true | |
| run: ./scripts/ci_check_no_file_changes.sh python | |
| - name: Setup test containers | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| cp dev-project/.env.template dev-project/.env | |
| docker compose -f dev-project/docker-compose.yml up -d | |
| - name: Wait for test containers to be ready | |
| if: steps.check.outcome == 'failure' | |
| timeout-minutes: 5 | |
| run: | | |
| until docker logs pipelinewise_dev | grep "PipelineWise Dev environment is ready" | |
| do | |
| echo 'Sleeping for 10s'; | |
| sleep 10; | |
| done | |
| - name: Run target snowflake end-to-end tests | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| docker exec -t \ | |
| -e TARGET_SNOWFLAKE_ACCOUNT=$TARGET_SNOWFLAKE_ACCOUNT \ | |
| -e TARGET_SNOWFLAKE_AWS_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_ACCESS_KEY \ | |
| -e TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY \ | |
| -e TARGET_SNOWFLAKE_DBNAME=$TARGET_SNOWFLAKE_DBNAME \ | |
| -e TARGET_SNOWFLAKE_FILE_FORMAT=$TARGET_SNOWFLAKE_FILE_FORMAT \ | |
| -e TARGET_SNOWFLAKE_PASSWORD=$TARGET_SNOWFLAKE_PASSWORD \ | |
| -e TARGET_SNOWFLAKE_S3_BUCKET=$TARGET_SNOWFLAKE_S3_BUCKET \ | |
| -e TARGET_SNOWFLAKE_S3_KEY_PREFIX=$TARGET_SNOWFLAKE_S3_KEY_PREFIX \ | |
| -e TARGET_SNOWFLAKE_SCHEMA=$TARGET_SNOWFLAKE_SCHEMA \ | |
| -e TARGET_SNOWFLAKE_STAGE=$TARGET_SNOWFLAKE_STAGE \ | |
| -e TARGET_SNOWFLAKE_USER=$TARGET_SNOWFLAKE_USER \ | |
| -e TARGET_SNOWFLAKE_WAREHOUSE=$TARGET_SNOWFLAKE_WAREHOUSE \ | |
| -e TAP_S3_CSV_AWS_KEY=$TAP_S3_CSV_AWS_KEY \ | |
| -e TAP_S3_CSV_AWS_SECRET_ACCESS_KEY=$TAP_S3_CSV_AWS_SECRET_ACCESS_KEY \ | |
| -e TAP_S3_CSV_BUCKET=$TAP_S3_CSV_BUCKET \ | |
| pipelinewise_dev pytest tests/end_to_end/target_snowflake/tap_mongodb -vx --timer-top-n 10 | |
| e2e_tests_other_targets: | |
| runs-on: ubuntu-20.04 | |
| environment: ci_tests | |
| steps: | |
| - name: Checking out repo | |
| uses: actions/checkout@v4.1.7 | |
| - name: Check if python changes are present | |
| id: check | |
| env: | |
| GITHUB_REPO: ${{ github.repository }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| continue-on-error: true | |
| run: ./scripts/ci_check_no_file_changes.sh python | |
| - name: Setup test containers | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| cp dev-project/.env.template dev-project/.env | |
| docker compose -f dev-project/docker-compose.yml up -d | |
| - name: Wait for test containers to be ready | |
| if: steps.check.outcome == 'failure' | |
| timeout-minutes: 5 | |
| run: | | |
| until docker logs pipelinewise_dev | grep "PipelineWise Dev environment is ready" | |
| do | |
| echo 'Sleeping for 10s'; | |
| sleep 10; | |
| done | |
| - name: Run target bigquery and redshift end-to-end tests | |
| if: steps.check.outcome == 'failure' | |
| run: | | |
| docker exec -t \ | |
| -e TAP_S3_CSV_AWS_KEY=$TAP_S3_CSV_AWS_KEY \ | |
| -e TAP_S3_CSV_AWS_SECRET_ACCESS_KEY=$TAP_S3_CSV_AWS_SECRET_ACCESS_KEY \ | |
| -e TAP_S3_CSV_BUCKET=$TAP_S3_CSV_BUCKET \ | |
| pipelinewise_dev pytest tests/end_to_end -vx --ignore=tests/end_to_end/test_target_postgres.py --ignore=tests/end_to_end/target_snowflake --timer-top-n 10 |