fix gh actions

re-data · Nov 12, 2023 · fe79238 · fe79238
1 parent 630a705
commit fe79238
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 32 deletions.
diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml
@@ -4,6 +4,7 @@ on: [push]
 
 env:
   DBT_PROFILES_DIR: ./
+  DBT_VERSION: 1.7
 
 jobs:
   generate-docs:
@@ -27,14 +28,13 @@ jobs:
       - name: Check out
         uses: actions/checkout@v2
 
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.7.x"
+          python-version: "3.8.x"
 
       - name: Install dependencies and run
         run: |
-          pip install dbt-core==1.3
-          pip install dbt-postgres==1.3
+          pip install dbt-postgres==$DBT_VERSION
           dbt deps
           dbt run
 
@@ -48,7 +48,7 @@ jobs:
 
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3
-        if: github.ref == 'refs/heads/main'
+        if: github.ref == 'refs/heads/main' && !env.ACT
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs
diff --git a/.github/workflows/run-db-tests.yml b/.github/workflows/run-db-tests.yml
@@ -8,9 +8,17 @@ env:
   RE_DATA_TESTING_USER: ${{ secrets.RE_DATA_TESTING_USER }}
   RE_DATA_TESTING_PASSWORD: ${{ secrets.RE_DATA_TESTING_PASSWORD }}
   REDSHIFT_RE_DATA_TESTING_HOST: ${{ secrets.REDSHIFT_RE_DATA_TESTING_HOST }}
+  BIGQUERY_TESTING_TYPE: ${{ secrets.BIGQUERY_TESTING_TYPE }}
   BIGQUERY_TESTING_PROJECT_ID: ${{ secrets.BIGQUERY_TESTING_PROJECT_ID }}
-  BIGQUERY_TESTING_SERVICE_ACCOUNT_BASE64: ${{ secrets.BIGQUERY_TESTING_SERVICE_ACCOUNT_BASE64 }}
-  BIGQUERY_TESTING_SERVICE_ACCOUNT_PATH: ${{ github.workspace }}/bigquery-testing-service-account.json
+  BIGQUERY_TESTING_PRIVATE_KEY_ID: ${{ secrets.BIGQUERY_TESTING_PRIVATE_KEY_ID }}
+  BIGQUERY_TESTING_PRIVATE_KEY: ${{ secrets.BIGQUERY_TESTING_PRIVATE_KEY }}
+  BIGQUERY_TESTING_CLIENT_EMAIL: ${{ secrets.BIGQUERY_TESTING_CLIENT_EMAIL }}
+  BIGQUERY_TESTING_CLIENT_ID: ${{ secrets.BIGQUERY_TESTING_CLIENT_ID }}
+  BIGQUERY_TESTING_AUTH_URI: ${{ secrets.BIGQUERY_TESTING_AUTH_URI }}
+  BIGQUERY_TESTING_TOKEN_URI: ${{ secrets.BIGQUERY_TESTING_TOKEN_URI }}
+  BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL }}
+  BIGQUERY_TESTING_CLIENT_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_CLIENT_X509_CERT_URL }}
+  DBT_VERSION: 1.7
 
 jobs:
   test-other-dbs:
@@ -24,9 +32,9 @@ jobs:
       - name: Check out
         uses: actions/checkout@v2
 
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.7.x"
+          python-version: "3.8.x"
 
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v3.x
@@ -40,16 +48,11 @@ jobs:
         run: |
           echo $DQ_SCHEMA
 
-      - name: Decode service account for BigQuery
-        if: matrix.database == 'bigquery'
-        run: 'echo $BIGQUERY_TESTING_SERVICE_ACCOUNT_BASE64 | base64 --decode > $BIGQUERY_TESTING_SERVICE_ACCOUNT_PATH'
-        shell: bash
-
       - name: Install dependencies
         working-directory: ./integration_tests
         run: |
           pip install -r requirements.txt
-          pip install dbt-${{ matrix.database }}==1.4.0
+          pip install dbt-${{ matrix.database }}==$DBT_VERSION
           dbt deps
 
       - name: Drop schemas
@@ -89,14 +92,15 @@ jobs:
       - name: Check out
         uses: actions/checkout@v2
 
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.7.x"
+          python-version: "3.8.x"
 
       - name: Install dependencies
         working-directory: ./integration_tests
         run: |
           pip install -r requirements.txt
+          pip install dbt-postgres==$DBT_VERSION
           dbt deps
 
       - name: Test DB
@@ -114,9 +118,9 @@ jobs:
       - name: Check out
         uses: actions/checkout@v2
 
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.7.x"
+          python-version: "3.8.x"
 
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v3.x
@@ -130,15 +134,10 @@ jobs:
         run: |
           echo $DQ_SCHEMA
 
-      - name: Decode service account for BigQuery
-        if: matrix.database == 'bigquery'
-        run: 'echo $BIGQUERY_TESTING_SERVICE_ACCOUNT_BASE64 | base64 --decode > $BIGQUERY_TESTING_SERVICE_ACCOUNT_PATH'
-        shell: bash
-
       - name: Install dependencies and drop branch schema
         working-directory: ./integration_tests
         run: |
           pip install -r requirements.txt
-          pip install dbt-${{ matrix.database }}==1.5.0
+          pip install dbt-${{ matrix.database }}==$DBT_VERSION
           dbt deps
           dbt run-operation drop_all_schemas --args "{ schema_name: ${{ env.DQ_SCHEMA }} }" --profile re_data_${{ matrix.database }} --vars "{ source_schema: ${{ env.DQ_SCHEMA }} }"
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,4 @@ logs/*
 dbt_modules/*
 dbt_packages/*
 target/*
+.env
diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml
@@ -2,4 +2,4 @@ packages:
   - local: ../
 
   - package: dbt-labs/dbt_utils
-    version: [">=1.0.0", "<1.1.0"]
+    version: [">=1.0.0", "<1.2.0"]
diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt
@@ -1,4 +1,2 @@
 pytest==6.2.5
 pyyaml==6.0
-dbt-core==1.4.0
-dbt-postgres==1.4.0
diff --git a/macros/public/store/generate_overview.sql b/macros/public/store/generate_overview.sql
@@ -86,7 +86,7 @@
     {% endset %}
 
     {% set overview_result = run_query(overview_query) %}
-    {% set overview_file_path = overview_path or 'target/re_data/overview.json' %}
+    {% set overview_file_path = overview_path or '../target/re_data/overview.json' %}
     {% do overview_result.to_json(overview_file_path) %}
     {{ save_monitored(monitored_path) }}
 

diff --git a/profiles.yml b/profiles.yml
@@ -39,12 +39,22 @@ re_data_bigquery:
   outputs:
     dev:
       type: bigquery
-      method: service-account
+      method: service-account-json
       project: "{{ env_var('BIGQUERY_TESTING_PROJECT_ID') }}"
       dataset: "{{ env_var('DQ_SCHEMA') }}"
       threads: 4
-      keyfile: "{{ env_var('BIGQUERY_TESTING_SERVICE_ACCOUNT_PATH') }}"
-      location: EU
+      keyfile_json:
+        type: "{{ env_var('BIGQUERY_TESTING_TYPE') }}"
+        project_id: "{{ env_var('BIGQUERY_TESTING_PROJECT_ID') }}"
+        private_key_id: "{{ env_var('BIGQUERY_TESTING_PRIVATE_KEY_ID') }}"
+        private_key: "{{ env_var('BIGQUERY_TESTING_PRIVATE_KEY') }}"
+        client_email: "{{ env_var('BIGQUERY_TESTING_CLIENT_EMAIL') }}"
+        client_id: "{{ env_var('BIGQUERY_TESTING_CLIENT_ID') }}"
+        auth_uri: "{{ env_var('BIGQUERY_TESTING_AUTH_URI') }}"
+        token_uri: "{{ env_var('BIGQUERY_TESTING_TOKEN_URI') }}"
+        auth_provider_x509_cert_url: "{{ env_var('BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL') }}"
+        client_x509_cert_url: "{{ env_var('BIGQUERY_TESTING_CLIENT_X509_CERT_URL') }}"
+      location: US
       timeout_seconds: 300
       priority: interactive
       retries: 1