refactor: sources async job #17429
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - main | |
| - "release/*" | |
| pull_request: | |
| jobs: | |
| integration: | |
| name: Integration | |
| runs-on: 'ubuntu-20.04' | |
| strategy: | |
| matrix: | |
| FEATURES: [ oss ,enterprise ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: '~1.21' | |
| check-latest: true | |
| - run: go version | |
| - run: go mod download # Not required, used to segregate module download vs test times | |
| - name: enterprise | |
| if: matrix.FEATURES == 'enterprise' | |
| run: go test -v ./integration_test/docker_test/docker_test.go -count 1 | |
| env: | |
| ENTERPRISE_TOKEN: ${{ secrets.ENTERPRISE_TOKEN }} | |
| - name: oss | |
| if: matrix.FEATURES == 'oss' | |
| run: go test -v ./integration_test/docker_test/docker_test.go -count 1 | |
| env: | |
| RSERVER_ENABLE_MULTITENANCY: ${{ matrix.MULTITENANCY }} | |
| warehouse-integration: | |
| name: Warehouse Integration | |
| runs-on: 'ubuntu-20.04' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| destination: [ bigquery, clickhouse, datalake, deltalake, mssql, azure-synapse, postgres, redshift, snowflake ] | |
| include: | |
| - package: warehouse/integrations/bigquery | |
| destination: bigquery | |
| - package: warehouse/integrations/clickhouse | |
| destination: clickhouse | |
| - package: warehouse/integrations/datalake | |
| destination: datalake | |
| - package: warehouse/integrations/deltalake | |
| destination: deltalake | |
| - package: warehouse/integrations/mssql | |
| destination: mssql | |
| - package: warehouse/integrations/azure-synapse | |
| destination: azure-synapse | |
| - package: warehouse/integrations/postgres | |
| destination: postgres | |
| - package: warehouse/integrations/redshift | |
| destination: redshift | |
| - package: warehouse/integrations/snowflake | |
| destination: snowflake | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: '~1.21' | |
| check-latest: true | |
| - run: go version | |
| - run: go mod download # Not required, used to segregate module download vs test times | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: rudderlabs | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Warehouse Service Integration [ ${{ matrix.destination }} ] | |
| run: make test-warehouse package=./${{ matrix.package }}/... | |
| env: | |
| BIGQUERY_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDENTIALS }} | |
| DATABRICKS_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.DATABRICKS_INTEGRATION_TEST_CREDENTIALS }} | |
| REDSHIFT_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.REDSHIFT_INTEGRATION_TEST_CREDENTIALS }} | |
| SNOWFLAKE_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDENTIALS }} | |
| SNOWFLAKE_RBAC_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.SNOWFLAKE_RBAC_INTEGRATION_TEST_CREDENTIALS }} | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.destination }} | |
| path: coverage.txt | |
| unit: | |
| name: Unit | |
| runs-on: 'ubuntu-20.04' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: '~1.21' | |
| check-latest: true | |
| - run: go version | |
| - run: go mod download # Not required, used to segregate module download vs test times | |
| - run: make test exclude="/rudder-server/(jobsdb|integration_test|processor|regulation-worker|router|services|suppression-backup-service|warehouse)" | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: unit | |
| path: coverage.txt | |
| package-unit: | |
| name: Package Unit | |
| runs-on: 'ubuntu-20.04' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: [ jobsdb, integration_test, processor, regulation-worker, router, services, suppression-backup-service, warehouse ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: '~1.21' | |
| check-latest: true | |
| - run: go version | |
| - run: go mod download | |
| - name: Package Unit [ ${{ matrix.package }} ] | |
| env: | |
| TEST_KAFKA_CONFLUENT_CLOUD_HOST: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_HOST }} | |
| TEST_KAFKA_CONFLUENT_CLOUD_KEY: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_KEY }} | |
| TEST_KAFKA_CONFLUENT_CLOUD_SECRET: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_SECRET }} | |
| TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST }} | |
| TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME }} | |
| TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING }} | |
| TEST_S3_DATALAKE_CREDENTIALS: ${{ secrets.TEST_S3_DATALAKE_CREDENTIALS }} | |
| BIGQUERY_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDENTIALS }} | |
| run: make test package=./${{ matrix.package }}/... | |
| - name: Upload coverage report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ matrix.package }}-unit | |
| path: coverage.txt | |
| coverage: | |
| name: Coverage | |
| runs-on: 'ubuntu-20.04' | |
| needs: | |
| - warehouse-integration | |
| - unit | |
| - package-unit | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: '~1.21' | |
| check-latest: true | |
| - name: Download coverage reports | |
| uses: actions/download-artifact@v3 | |
| - name: Merge Coverage | |
| run: | | |
| go install github.com/wadey/gocovmerge@latest | |
| gocovmerge */coverage.txt > coverage.txt | |
| - uses: codecov/codecov-action@v3 | |
| with: | |
| fail_ci_if_error: true | |
| files: ./coverage.txt | |
| all-green: | |
| name: All | |
| if: always() | |
| runs-on: ubuntu-latest | |
| needs: | |
| - integration | |
| - warehouse-integration | |
| - unit | |
| - package-unit | |
| steps: | |
| - uses: re-actors/alls-green@v1.2.2 | |
| with: | |
| jobs: ${{ toJSON(needs) }} |