ci: configure env vars for CI in .env file

Fixes apache#1545.
lidavidm · Mar 1, 2024 · 00e7b56 · 00e7b56
1 parent 62b0728
commit 00e7b56
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 55 deletions.
diff --git a/.env b/.env
@@ -41,3 +41,18 @@ DOTNET=7.0
 # When updating, also update the docs, which list the version of libpq/SQLite
 # that vcpkg (and hence our wheels) ship
 VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"
+
+# These are used to tell tests where to find services for integration testing.
+# They are valid if the services are started with the docker-compose config.
+ADBC_DREMIO_FLIGHTSQL_PASS=dremio123
+ADBC_DREMIO_FLIGHTSQL_URI=grpc+tcp://localhost:32010
+ADBC_DREMIO_FLIGHTSQL_USER=dremio
+ADBC_JDBC_MSSQL_URL: "localhost:1433"
+ADBC_JDBC_MSSQL_USER: "SA"
+ADBC_JDBC_MSSQL_PASSWORD: "Password1!"
+ADBC_JDBC_POSTGRESQL_URL: "localhost:5432/postgres"
+ADBC_JDBC_POSTGRESQL_USER: "postgres"
+ADBC_JDBC_POSTGRESQL_PASSWORD: "password"
+ADBC_POSTGRESQL_TEST_URI=postgresql://localhost:5432/postgres?user=postgres&password=password
+ADBC_SQLITE_FLIGHTSQL_URI=grpc+tcp://localhost:8080
+ADBC_TEST_FLIGHTSQL_URI=grpc+tcp://localhost:41414
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -138,45 +138,31 @@ jobs:
       - name: Start SQLite server and Dremio
         shell: bash -l {0}
         run: |
-          docker-compose up -d flightsql-test flightsql-sqlite-test dremio dremio-init
+          docker-compose up --wait --detach flightsql-test flightsql-sqlite-test dremio dremio-init
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
 
       - name: Build FlightSQL Driver
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
+          BUILD_DRIVER_MANAGER: "1"
         run: |
           ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
+          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
       - name: Test FlightSQL Driver against Dremio and SQLite
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
-          ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
-          ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
-          ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
-          ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
-          ADBC_TEST_FLIGHTSQL_URI: "grpc+tcp://localhost:41414"
         run: |
           ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
           ./ci/scripts/cpp_test.sh "$(pwd)/build"
-      - name: Build Python Flight SQL driver
-        shell: bash -l {0}
-        env:
-          BUILD_ALL: "0"
-          BUILD_DRIVER_FLIGHTSQL: "1"
-          BUILD_DRIVER_MANAGER: "1"
-        run: |
-          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
       - name: Test Python Flight SQL driver against Dremio
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
-          ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
-          ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
-          ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
-          ADBC_TEST_FLIGHTSQL_URI: "grpc+tcp://localhost:41414"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build"
@@ -220,17 +206,15 @@ jobs:
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
+          BUILD_DRIVER_MANAGER: "1"
           BUILD_DRIVER_POSTGRESQL: "1"
         run: |
           ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
-      - name: Build Python PostgreSQL Driver
+          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
+      - name: Configure Integration Env Vars
         shell: bash -l {0}
-        env:
-          BUILD_ALL: "0"
-          BUILD_DRIVER_MANAGER: "1"
-          BUILD_DRIVER_POSTGRESQL: "1"
         run: |
-          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
 
       - name: Test PostgreSQL Driver - postgres 11
         shell: bash -l {0}
@@ -239,7 +223,6 @@ jobs:
           BUILD_DRIVER_POSTGRESQL: "1"
           ADBC_USE_ASAN: "ON"
           ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=11 docker compose up --wait --detach postgres-test
@@ -251,9 +234,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=12 docker compose up --wait --detach postgres-test
@@ -265,9 +245,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=13 docker compose up --wait --detach postgres-test
@@ -279,9 +256,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=14 docker compose up --wait --detach postgres-test
@@ -293,9 +267,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=15 docker compose up --wait --detach postgres-test
@@ -307,9 +278,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=16 docker compose up --wait --detach postgres-test

diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -57,16 +57,9 @@ jobs:
       - name: Start test services
         shell: bash -l {0}
         run: |
-          docker compose up -d --wait flightsql-sqlite-test mssql-test postgres-test
+          docker compose up --detach --wait flightsql-sqlite-test mssql-test postgres-test
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
       - name: Build/Test
-        env:
-          ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
-          ADBC_JDBC_MSSQL_URL: "localhost:1433"
-          ADBC_JDBC_MSSQL_USER: "SA"
-          ADBC_JDBC_MSSQL_PASSWORD: "Password1!"
-          ADBC_JDBC_POSTGRESQL_URL: "localhost:5432/postgres"
-          ADBC_JDBC_POSTGRESQL_USER: "postgres"
-          ADBC_JDBC_POSTGRESQL_PASSWORD: "password"
         run: |
           cd java
           mvn install

diff --git a/.github/workflows/native-unix.yml b/.github/workflows/native-unix.yml
@@ -572,18 +572,16 @@ jobs:
           retention-days: 2
           path: |
             docs/build/html
+      - name: Configure Integration Env Vars
+        shell: bash -l {0}
+        run: |
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
       - name: Test Recipes (C++)
         shell: bash -l {0}
         run: |
           ./ci/scripts/cpp_recipe.sh $(pwd) ~/local build/recipe
       - name: Test Recipes (Python)
         shell: bash -l {0}
-        env:
-          ADBC_POSTGRESQL_TEST_URI: "postgres://localhost:5432/postgres?user=postgres&password=password"
-          ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
-          ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
-          ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
-          ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
         run: |
           docker compose up --detach --wait dremio dremio-init flightsql-sqlite-test postgres-test
           pytest -vvs docs/source/tests/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -54,6 +54,27 @@ develop and support Conda users.
 [conda]: https://docs.conda.io/en/latest/
 [mambaforge]: https://mamba.readthedocs.io/en/latest/installation.html
 
+### Running Integration Tests
+
+Many of the test suites need to run against external services.  For example,
+the PostgreSQL driver needs to test against a running database!  This can be
+done by setting environment variables to tell tests where the services they
+need can be located.
+
+To standardize the configuration of these services, we use a Docker Compose
+file and a dotenv file.  Services can be started with Docker Compose:
+
+```shell
+$ docker compose up --detach --wait postgres-test
+```
+
+Then, source the .env file at the root of the repo to set the environment
+variables before running tests:
+
+```shell
+$ source .env
+```
+
 ### C/C++
 
 All libraries here contained within one CMake project. To build any