Merge branch 'main' into dev/ryansyed/getobjects_tableconstraints

ryan-syed · Mar 19, 2024 · 7ed0fbc · 7ed0fbc
2 parents 1426363 + 8768cd2
commit 7ed0fbc
Show file tree

Hide file tree

Showing 158 changed files with 7,115 additions and 1,780 deletions.
diff --git a/.env b/.env
@@ -31,7 +31,7 @@ ARCH_CONDA_FORGE=linux_64_
 JDK=8
 MANYLINUX=2014
 MAVEN=3.5.4
-PYTHON=3.10
+PYTHON=3.8
 GO=1.19.5
 ARROW_MAJOR_VERSION=14
 DOTNET=7.0
@@ -40,4 +40,20 @@ DOTNET=7.0
 # ci/scripts/install_vcpkg.sh script. Keep in sync with apache/arrow .env.
 # When updating, also update the docs, which list the version of libpq/SQLite
 # that vcpkg (and hence our wheels) ship
-VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"
+VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6"
+
+# These are used to tell tests where to find services for integration testing.
+# They are valid if the services are started with the docker-compose config.
+ADBC_DREMIO_FLIGHTSQL_PASS=dremio123
+ADBC_DREMIO_FLIGHTSQL_URI=grpc+tcp://localhost:32010
+ADBC_DREMIO_FLIGHTSQL_USER=dremio
+ADBC_JDBC_MSSQL_URL=localhost:1433
+ADBC_JDBC_MSSQL_USER=SA
+ADBC_JDBC_MSSQL_PASSWORD=Password1!
+ADBC_JDBC_POSTGRESQL_URL=localhost:5432/postgres
+ADBC_JDBC_POSTGRESQL_USER=postgres
+ADBC_JDBC_POSTGRESQL_PASSWORD=password
+ADBC_JDBC_POSTGRESQL_DATABASE=postgres
+ADBC_POSTGRESQL_TEST_URI=postgresql://localhost:5432/postgres?user=postgres&password=password
+ADBC_SQLITE_FLIGHTSQL_URI=grpc+tcp://localhost:8080
+ADBC_TEST_FLIGHTSQL_URI=grpc+tcp://localhost:41414
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -23,6 +23,18 @@ updates:
       interval: "weekly"
     commit-message:
       prefix: "chore: "
+  - package-ecosystem: "gomod"
+    directory: "/go/adbc/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore(go/adbc): "
+  - package-ecosystem: "maven"
+    directory: "/java/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "chore(java): "
   - package-ecosystem: "nuget"
     directory: "/csharp/"
     schedule:

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -78,6 +78,10 @@ jobs:
         run: |
           mamba install -c conda-forge \
             --file ci/conda_env_cpp.txt
+      - name: Work around ASAN issue (GH-1617)
+        shell: bash -l {0}
+        run: |
+          sudo sysctl vm.mmap_rnd_bits=28
       - name: Build DuckDB Integration Tests
         shell: bash -l {0}
         env:
@@ -135,55 +139,45 @@ jobs:
           check-latest: true
           cache: true
           cache-dependency-path: go/adbc/go.sum
+      - name: Work around ASAN issue (GH-1617)
+        shell: bash -l {0}
+        run: |
+          sudo sysctl vm.mmap_rnd_bits=28
       - name: Start SQLite server and Dremio
         shell: bash -l {0}
         run: |
-          docker-compose up -d flightsql-test flightsql-sqlite-test dremio dremio-init
+          docker compose up --detach flightsql-test flightsql-sqlite-test dremio dremio-init
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
 
       - name: Build FlightSQL Driver
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
+          BUILD_DRIVER_MANAGER: "1"
         run: |
           ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
+          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
       - name: Test FlightSQL Driver against Dremio and SQLite
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
-          ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
-          ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
-          ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
-          ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
-          ADBC_TEST_FLIGHTSQL_URI: "grpc+tcp://localhost:41414"
         run: |
           ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
           ./ci/scripts/cpp_test.sh "$(pwd)/build"
-      - name: Build Python Flight SQL driver
-        shell: bash -l {0}
-        env:
-          BUILD_ALL: "0"
-          BUILD_DRIVER_FLIGHTSQL: "1"
-          BUILD_DRIVER_MANAGER: "1"
-        run: |
-          ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
       - name: Test Python Flight SQL driver against Dremio
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_FLIGHTSQL: "1"
-          ADBC_DREMIO_FLIGHTSQL_URI: "grpc+tcp://localhost:32010"
-          ADBC_DREMIO_FLIGHTSQL_USER: "dremio"
-          ADBC_DREMIO_FLIGHTSQL_PASS: "dremio123"
-          ADBC_TEST_FLIGHTSQL_URI: "grpc+tcp://localhost:41414"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build"
       - name: Stop SQLite server and Dremio
         shell: bash -l {0}
         run: |
-          docker-compose down
+          docker compose down
 
   postgresql:
     name: "PostgreSQL Integration Tests"
@@ -216,30 +210,31 @@ jobs:
             --file ci/conda_env_cpp.txt \
             --file ci/conda_env_python.txt
           pip install pytest-error-for-skips
-      - name: Build PostgreSQL Driver
+      - name: Work around ASAN issue (GH-1617)
         shell: bash -l {0}
-        env:
-          BUILD_ALL: "0"
-          BUILD_DRIVER_POSTGRESQL: "1"
         run: |
-          ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
-      - name: Build Python PostgreSQL Driver
+          sudo sysctl vm.mmap_rnd_bits=28
+      - name: Build PostgreSQL Driver
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_MANAGER: "1"
           BUILD_DRIVER_POSTGRESQL: "1"
         run: |
+          ./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
           ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
+      - name: Configure Integration Env Vars
+        shell: bash -l {0}
+        run: |
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
+          echo "ADBC_USE_ASAN=ON" >> $GITHUB_ENV
+          echo "ADBC_USE_UBSAN=ON" >> $GITHUB_ENV
 
       - name: Test PostgreSQL Driver - postgres 11
         shell: bash -l {0}
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=11 docker compose up --wait --detach postgres-test
@@ -251,9 +246,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=12 docker compose up --wait --detach postgres-test
@@ -265,9 +257,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=13 docker compose up --wait --detach postgres-test
@@ -279,9 +268,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=14 docker compose up --wait --detach postgres-test
@@ -293,9 +279,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=15 docker compose up --wait --detach postgres-test
@@ -307,9 +290,6 @@ jobs:
         env:
           BUILD_ALL: "0"
           BUILD_DRIVER_POSTGRESQL: "1"
-          ADBC_USE_ASAN: "ON"
-          ADBC_USE_UBSAN: "ON"
-          ADBC_POSTGRESQL_TEST_URI: "postgresql://localhost:5432/postgres?user=postgres&password=password"
           PYTEST_ADDOPTS: "--error-for-skips"
         run: |
           env POSTGRES_VERSION=16 docker compose up --wait --detach postgres-test
@@ -347,6 +327,10 @@ jobs:
           mamba install -c conda-forge \
             --file ci/conda_env_cpp.txt \
             --file ci/conda_env_python.txt
+      - name: Work around ASAN issue (GH-1617)
+        shell: bash -l {0}
+        run: |
+          sudo sysctl vm.mmap_rnd_bits=28
       - uses: actions/setup-go@v5
         with:
           go-version: 1.19.13

diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -57,16 +57,9 @@ jobs:
       - name: Start test services
         shell: bash -l {0}
         run: |
-          docker compose up -d --wait flightsql-sqlite-test mssql-test postgres-test
+          docker compose up --detach --wait flightsql-sqlite-test mssql-test postgres-test
+          cat .env | grep -v -e '^#' | awk NF | tee -a $GITHUB_ENV
       - name: Build/Test
-        env:
-          ADBC_SQLITE_FLIGHTSQL_URI: "grpc+tcp://localhost:8080"
-          ADBC_JDBC_MSSQL_URL: "localhost:1433"
-          ADBC_JDBC_MSSQL_USER: "SA"
-          ADBC_JDBC_MSSQL_PASSWORD: "Password1!"
-          ADBC_JDBC_POSTGRESQL_URL: "localhost:5432/postgres"
-          ADBC_JDBC_POSTGRESQL_USER: "postgres"
-          ADBC_JDBC_POSTGRESQL_PASSWORD: "password"
         run: |
           cd java
           mvn install
@@ -104,4 +97,5 @@ jobs:
           --add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
           --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
           HERE
-          mvn -P errorprone install
+          # Our linters are compile-time, no point re-running tests
+          mvn -P errorprone install -DskipTests