refactor(offer_categorization): remove files and clean up codebase #1118
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "2. [on_ci] Integration Workflow" | |
| on: | |
| pull_request: | |
| permissions: | |
| contents: write | |
| id-token: write | |
| jobs: | |
| linter: | |
| uses: ./.github/workflows/reusable_linter.yml | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| find-dbt-changes: | |
| outputs: | |
| changes: ${{ steps.dbt-changes.outputs.changes }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "is dbt project modified?" | |
| id: dbt-changes | |
| env: | |
| TARGET_BRANCH: ${{ github.base_ref }} | |
| run: .github/workflows/scripts/check_dbt_changes.sh $TARGET_BRANCH | |
| dbt-integration-prod: | |
| needs: find-dbt-changes | |
| if: github.base_ref == 'production' && needs.find-dbt-changes.outputs.changes == 'true' | |
| with: | |
| TARGET_COMPOSER_BUCKET: ${{ vars.COMPOSER_DAGS_BUCKET_PROD }} | |
| ENV_SHORT_NAME: "prod" | |
| APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_PROD }} | |
| uses: ./.github/workflows/reusable_dbt_workflow.yml | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| dbt-integration-stg: | |
| needs: find-dbt-changes | |
| if: github.base_ref == 'master' && needs.find-dbt-changes.outputs.changes == 'true' | |
| with: | |
| TARGET_COMPOSER_BUCKET: ${{ vars.COMPOSER_DAGS_BUCKET_STG }} | |
| ENV_SHORT_NAME: "stg" | |
| APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_STG }} | |
| uses: ./.github/workflows/reusable_dbt_workflow.yml | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| find-test-jobs: | |
| outputs: | |
| jobs: ${{ steps.compute.outputs.jobs }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: "Get etl_jobs changed files" | |
| id: "changed-etl-files" | |
| uses: tj-actions/changed-files@v41 | |
| with: | |
| files: "jobs/etl_jobs/**" | |
| json: true | |
| dir_names: true | |
| dir_names_max_depth: 4 | |
| - name: "Get ml_jobs changed files" | |
| id: "changed-ml-files" | |
| uses: tj-actions/changed-files@v41 | |
| with: | |
| files: "jobs/ml_jobs/**" | |
| json: true | |
| dir_names: true | |
| dir_names_max_depth: 5 | |
| - name: "Trouver les Jobs testables" | |
| id: "find-testable-jobs" | |
| run: .github/workflows/scripts/find_tests.sh | |
| - name: "Show found files" | |
| id: "compute" | |
| run: | | |
| output=$(python3 .github/workflows/scripts/find_test_intersections.py ${{ steps.changed-etl-files.outputs.all_changed_files }} ${{ steps.changed-ml-files.outputs.all_changed_files }} ${{ steps.find-testable-jobs.outputs.testable_jobs }}) | |
| echo "Found tests: $output" | |
| echo "$output" >> $GITHUB_OUTPUT | |
| assert-matrix-not-empty: | |
| needs: find-test-jobs | |
| if: ${{ needs.find-test-jobs.outputs.jobs != '' && toJson(fromJson(needs.find-test-jobs.outputs.jobs)) != '[]' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "Matrix Not Empty" | |
| test-jobs: | |
| uses: ./.github/workflows/reusable_job_test.yml | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} | |
| needs: [assert-matrix-not-empty, find-test-jobs] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: ${{ fromJSON(needs.find-test-jobs.outputs.jobs) }} | |
| with: | |
| JOB_PATH: ${{ matrix.job }} | |
| find-changed-orchestration: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run_tests: ${{ steps.changed-orchestration-files.outputs.any_modified }} | |
| steps: | |
| - name: Get changed files in orchestration folder | |
| id: changed-orchestration-files | |
| uses: tj-actions/changed-files@v41 | |
| with: | |
| files: "orchestration/**" | |
| test-orchestration: | |
| needs: find-changed-orchestration | |
| if: needs.find-changed-orchestration.outputs.run_tests == 'true' | |
| uses: ./.github/workflows/reusable_test_orchestration.yml | |
| secrets: | |
| GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }} | |
| GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }} |