Skip to content
Native Libraries (Unix)

fix(ci): update website_build.sh for new versioning scheme #975

Sign in to view logs

fix(ci): update website_build.sh for new versioning scheme

fix(ci): update website_build.sh for new versioning scheme #975

Workflow file for this run

.github/workflows/native-unix.yml at 2aca79a
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Native Libraries (Unix)
on:
pull_request:
branches:
- main
paths:
- "adbc.h"
- "c/**"
- "ci/**"
- "docs/**"
- "glib/**"
- "go/**"
- "python/**"
- "ruby/**"
- ".github/workflows/native-unix.yml"
push:
paths:
- "adbc.h"
- "c/**"
- "ci/**"
- "docs/**"
- "glib/**"
- "go/**"
- "python/**"
- "ruby/**"
- ".github/workflows/native-unix.yml"
workflow_call:
concurrency:
group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}-unix
cancel-in-progress: true
permissions:
contents: read
env:
# Increment this to reset cache manually
CACHE_NUMBER: "1"
jobs:
# ------------------------------------------------------------
# Common build (builds libraries used in GLib, Python, Ruby)
# ------------------------------------------------------------
drivers-build-conda:
name: "Common Libraries (Conda/${{ matrix.os }})"
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: ["macos-13", "macos-latest", "ubuntu-latest"]
include:
- os: macos-13
goarch: x64
- os: macos-latest
goarch: arm64
- os: ubuntu-latest
goarch: x64
env:
# Required for macOS
# https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk
CXXFLAGS: "-D_LIBCPP_DISABLE_AVAILABILITY"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Get required Go version
run: |
(. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
- name: Get Date
id: get-date
shell: bash
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache@v4
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
shell: bash -l {0}
run: |
mamba install -c conda-forge \
--file ci/conda_env_cpp.txt
- uses: actions/setup-go@v5
with:
go-version: "${{ env.GO_VERSION }}"
check-latest: true
cache: true
cache-dependency-path: go/adbc/go.sum
- name: Build and Install (No ASan)
shell: bash -l {0}
run: |
# Python and others need something that don't use the ASAN runtime
rm -rf "$(pwd)/build"
export BUILD_ALL=1
export ADBC_BUILD_TESTS=OFF
export ADBC_USE_ASAN=OFF
export ADBC_USE_UBSAN=OFF
export PATH=$RUNNER_TOOL_CACHE/go/${GO_VERSION}/${{ matrix.goarch }}/bin:$PATH
export ADBC_CMAKE_ARGS="-DCMAKE_UNITY_BUILD=ON"
./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Go Build
shell: bash -l {0}
env:
CGO_ENABLED: "1"
run: |
export PATH=$RUNNER_TOOL_CACHE/go/${GO_VERSION}/${{ matrix.goarch }}/bin:$PATH
./ci/scripts/go_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- uses: actions/upload-artifact@v4
with:
name: driver-manager-${{ matrix.os }}
retention-days: 3
path: |
~/local
# ------------------------------------------------------------
# C/C++ (builds and tests)
# ------------------------------------------------------------
drivers-test-conda:
name: "C/C++ (Conda/${{ matrix.os }})"
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: ["macos-13", "macos-latest", "ubuntu-latest"]
env:
# Required for macOS
# https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk
CXXFLAGS: "-D_LIBCPP_DISABLE_AVAILABILITY"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Get Date
id: get-date
shell: bash
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache@v4
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
shell: bash -l {0}
run: |
mamba install -c conda-forge \
--file ci/conda_env_cpp.txt
- name: Work around ASAN issue (GH-1617)
shell: bash -l {0}
if: matrix.os == 'ubuntu-latest'
run: |
sudo sysctl vm.mmap_rnd_bits=28
- name: Build
shell: bash -l {0}
env:
ADBC_CMAKE_ARGS: "-DCMAKE_UNITY_BUILD=ON"
BUILD_ALL: "0"
BUILD_DRIVER_MANAGER: "1"
BUILD_DRIVER_POSTGRESQL: "1"
BUILD_DRIVER_SQLITE: "1"
run: |
./ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build"
- name: Test
shell: bash -l {0}
env:
BUILD_ALL: "0"
BUILD_DRIVER_MANAGER: "1"
# PostgreSQL is explicitly not tested here since we don't spawn
# Postgres; see integration.yml
BUILD_DRIVER_POSTGRESQL: "0"
BUILD_DRIVER_SQLITE: "1"
run: |
./ci/scripts/cpp_test.sh "$(pwd)/build"
drivers-test-meson:
name: "Meson - C/C++ (Conda/${{ matrix.os }})"
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: ["ubuntu-latest"]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Install Dependencies
run: |
sudo apt update
sudo apt install -y libpq-dev ninja-build
- name: Get required Go version
run: |
(. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
- uses: actions/setup-go@v5
with:
go-version: "${{ env.GO_VERSION }}"
check-latest: true
cache: true
cache-dependency-path: go/adbc/go.sum
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: Install Meson via Python
run: pip install meson
- name: Start SQLite server, Dremio, and postgresql
shell: bash -l {0}
run: |
env POSTSGRES_VERSION=16 docker compose up --detach --wait \
dremio \
dremio-init \
flightsql-test \
flightsql-sqlite-test \
postgres-test
pip install python-dotenv[cli]
python -m dotenv -f .env list --format simple | tee -a $GITHUB_ENV
- name: Build
run: |
meson setup \
-Db_sanitize=address,undefined \
-Ddriver_manager=true \
-Dflightsql=true \
-Dpostgresql=true \
-Dsnowflake=true \
-Dsqlite=true \
-Dtests=true \
c c/build
meson compile -C c/build
- name: Test
run: |
meson test -C c/build --print-errorlogs
- name: Stop SQLite server, Dremio, and postgresql
shell: bash -l {0}
run: |
docker compose down
clang-tidy-conda:
name: "clang-tidy"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Get Date
id: get-date
shell: bash
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache@v4
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
shell: bash -l {0}
run: |
mamba install -c conda-forge \
--file ci/conda_env_cpp.txt \
--file ci/conda_env_cpp_lint.txt
- name: clang-tidy
shell: bash -l {0}
run: |
./ci/scripts/cpp_clang_tidy.sh "$(pwd)" "$(pwd)/build"
# ------------------------------------------------------------
# GLib/Ruby
# ------------------------------------------------------------
glib-conda:
name: "GLib/Ruby (Conda/${{ matrix.os }})"
runs-on: ${{ matrix.os }}
needs:
- drivers-build-conda
strategy:
matrix:
# N.B. no macos-latest here since conda-forge does not package
# arrow-c-glib for M1
os: ["macos-13", "ubuntu-latest"]
env:
# Required for macOS
# https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk
CXXFLAGS: "-D_LIBCPP_DISABLE_AVAILABILITY"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Get Date
id: get-date
shell: bash
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache@v4
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
shell: bash -l {0}
run: |
mamba install -c conda-forge \
'arrow-c-glib>=10.0.1' \
--file ci/conda_env_cpp.txt \
--file ci/conda_env_glib.txt
- name: Work around ASAN issue (GH-1617)
shell: bash -l {0}
if: matrix.os == 'ubuntu-latest'
run: |
sudo sysctl vm.mmap_rnd_bits=28
- uses: actions/download-artifact@v4
with:
name: driver-manager-${{ matrix.os }}
path: ~/local
- name: Build GLib Driver Manager
shell: bash -l {0}
run: |
env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 ./ci/scripts/glib_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Test GLib/Ruby Driver Manager
shell: bash -l {0}
run: |
env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 ./ci/scripts/glib_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
# ------------------------------------------------------------
# Go
# ------------------------------------------------------------
go-no-cgo:
name: "Go (No CGO) (${{ matrix.os }})"
env:
CGO_ENABLED: "0"
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: ["macos-13", "macos-latest", "ubuntu-latest", "windows-latest"]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-go@v5
with:
go-version-file: "go/adbc/go.mod"
check-latest: true
cache: true
cache-dependency-path: go/adbc/go.sum
- name: Work around ASAN issue (GH-1617)
shell: bash -l {0}
if: matrix.os == 'ubuntu-latest'
run: |
sudo sysctl vm.mmap_rnd_bits=28
- name: Install staticcheck
run: go install honnef.co/go/tools/cmd/staticcheck@latest
- name: Go Build
run: |
./ci/scripts/go_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Run Staticcheck
run: |
pushd go/adbc
staticcheck -f stylish ./...
popd
- name: Go Test
env:
SNOWFLAKE_DATABASE: ADBC_TESTING
SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
run: |
./ci/scripts/go_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
go-conda:
name: "Go (CGO) (Conda/${{ matrix.os }})"
runs-on: ${{ matrix.os }}
needs:
- drivers-build-conda
strategy:
matrix:
os: ["macos-13", "macos-latest", "ubuntu-latest"]
include:
- os: macos-13
goarch: x64
- os: macos-latest
goarch: arm64
- os: ubuntu-latest
goarch: x64
env:
CGO_ENABLED: "1"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Get required Go version
run: |
(. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
- name: Get Date
id: get-date
shell: bash
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache@v4
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Work around ASAN issue (GH-1617)
shell: bash -l {0}
if: matrix.os == 'ubuntu-latest'
run: |
sudo sysctl vm.mmap_rnd_bits=28
- name: Install Dependencies
shell: bash -l {0}
run: |
mamba install -c conda-forge \
--file ci/conda_env_cpp.txt
- uses: actions/setup-go@v5
with:
go-version: "${{ env.GO_VERSION }}"
check-latest: true
cache: true
cache-dependency-path: go/adbc/go.sum
- name: Install staticcheck
shell: bash -l {0}
if: matrix.os == 'ubuntu-latest'
run: go install honnef.co/go/tools/cmd/staticcheck@latest
- uses: actions/download-artifact@v4
with:
name: driver-manager-${{ matrix.os }}
path: ~/local
- name: Go Build
shell: bash -l {0}
run: |
export PATH=$RUNNER_TOOL_CACHE/go/${GO_VERSION}/${{ matrix.goarch }}/bin:$PATH
./ci/scripts/go_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Run Staticcheck
if: matrix.os == 'ubuntu-latest'
shell: bash -l {0}
run: |
pushd go/adbc
staticcheck -f stylish ./...
popd
- name: Go Test
shell: bash -l {0}
env:
SNOWFLAKE_URI: ${{ secrets.SNOWFLAKE_URI }}
run: |
export PATH=$RUNNER_TOOL_CACHE/go/${GO_VERSION}/${{ matrix.goarch }}/bin:$PATH
./ci/scripts/go_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
# ------------------------------------------------------------
# Python/doctests
# ------------------------------------------------------------
python-conda:
name: "Python ${{ matrix.python }} (Conda/${{ matrix.os }})"
runs-on: ${{ matrix.os }}
needs:
- drivers-build-conda
strategy:
matrix:
os: ["macos-13", "macos-latest", "ubuntu-latest"]
python: ["3.9", "3.11"]
env:
# Required for macOS
# https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk
CXXFLAGS: "-D_LIBCPP_DISABLE_AVAILABILITY"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Get required Go version
run: |
(. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
- name: Get Date
id: get-date
shell: bash
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache@v4
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
shell: bash -l {0}
run: |
mamba install -c conda-forge \
python=${{ matrix.python }} \
--file ci/conda_env_cpp.txt \
--file ci/conda_env_python.txt
- name: Work around ASAN issue (GH-1617)
shell: bash -l {0}
if: matrix.os == 'ubuntu-latest'
run: |
sudo sysctl vm.mmap_rnd_bits=28
- uses: actions/setup-go@v5
with:
go-version: "${{ env.GO_VERSION }}"
check-latest: true
cache: true
cache-dependency-path: go/adbc/go.sum
- uses: actions/download-artifact@v4
with:
name: driver-manager-${{ matrix.os }}
path: ~/local
- name: Build
shell: bash -l {0}
env:
BUILD_ALL: "1"
ADBC_USE_ASAN: "0"
ADBC_USE_UBSAN: "0"
run: |
export PATH=$RUNNER_TOOL_CACHE/go/${GO_VERSION}/x64/bin:$PATH
./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Build Panic Dummy
shell: bash -l {0}
run: |
export PATH=$RUNNER_TOOL_CACHE/go/${GO_VERSION}/x64/bin:$PATH
if [[ $(uname) = "Darwin" ]]; then
make -C ./go/adbc/pkg libadbc_driver_panicdummy.dylib
else
make -C ./go/adbc/pkg libadbc_driver_panicdummy.so
fi
- name: Test Python Driver Manager
shell: bash -l {0}
run: |
if [[ $(uname) = "Darwin" ]]; then
export PANICDUMMY_LIBRARY_PATH=$(pwd)/go/adbc/pkg/libadbc_driver_panicdummy.dylib
else
export PANICDUMMY_LIBRARY_PATH=$(pwd)/go/adbc/pkg/libadbc_driver_panicdummy.so
fi
export PATH=$RUNNER_TOOL_CACHE/go/${GO_VERSION}/x64/bin:$PATH
env BUILD_ALL=0 BUILD_DRIVER_MANAGER=1 ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Test Python Driver Flight SQL
shell: bash -l {0}
run: |
export PATH=$RUNNER_TOOL_CACHE/go/${GO_VERSION}/x64/bin:$PATH
# Can't use Docker on macOS
pushd $(pwd)/go/adbc
go build -o testserver ./driver/flightsql/cmd/testserver
popd
$(pwd)/go/adbc/testserver -host 0.0.0.0 -port 41414 &
while ! curl --http2-prior-knowledge -H "content-type: application/grpc" -v localhost:41414 -XPOST;
do
echo "Waiting for test server..."
jobs
sleep 5
done
export ADBC_TEST_FLIGHTSQL_URI=grpc://localhost:41414
env BUILD_ALL=0 BUILD_DRIVER_FLIGHTSQL=1 ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
kill %1
- name: Test Python Driver PostgreSQL
shell: bash -l {0}
run: |
env BUILD_ALL=0 BUILD_DRIVER_POSTGRESQL=1 ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Test Python Driver SQLite
shell: bash -l {0}
run: |
env BUILD_ALL=0 BUILD_DRIVER_SQLITE=1 ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Test Python Driver Snowflake
shell: bash -l {0}
run: |
env BUILD_ALL=0 BUILD_DRIVER_SNOWFLAKE=1 ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
- name: Typecheck Python
shell: bash -l {0}
run: |
./ci/scripts/python_typecheck.sh "$(pwd)"
python-docs:
name: "Documentation ${{ matrix.python }} (Conda/${{ matrix.os }})"
runs-on: ${{ matrix.os }}
needs:
- drivers-build-conda
strategy:
matrix:
os: ["ubuntu-latest"]
python: ["3.11"]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Get Date
id: get-date
shell: bash
run: |
echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
- name: Cache Conda
uses: actions/cache@v4
with:
path: ~/conda_pkgs_dir
key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
use-only-tar-bz2: false
use-mamba: true
- name: Install Dependencies
shell: bash -l {0}
run: |
mamba install -c conda-forge \
python=${{ matrix.python }} \
--file ci/conda_env_docs.txt \
--file ci/conda_env_python.txt
- name: Work around ASAN issue (GH-1617)
shell: bash -l {0}
if: matrix.os == 'ubuntu-latest'
run: |
sudo sysctl vm.mmap_rnd_bits=28
- uses: actions/download-artifact@v4
with:
name: driver-manager-${{ matrix.os }}
path: ~/local
- name: Build Python
shell: bash -l {0}
env:
BUILD_ALL: "1"
ADBC_USE_ASAN: "0"
ADBC_USE_UBSAN: "0"
run: |
./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" "$HOME/local"
# Docs requires Python packages since it runs doctests
- name: Build Docs
shell: bash -l {0}
run: |
./ci/scripts/docs_build.sh "$(pwd)"
- name: Archive docs
uses: actions/upload-artifact@v4
with:
name: docs
retention-days: 2
path: |
docs/build/html
- name: Configure Integration Env Vars
shell: bash -l {0}
run: |
pip install python-dotenv[cli]
python -m dotenv -f .env list --format simple | tee -a $GITHUB_ENV
- name: Test Recipes (C++)
shell: bash -l {0}
run: |
./ci/scripts/cpp_recipe.sh $(pwd) ~/local build/recipe
- name: Test Recipes (Python)
shell: bash -l {0}
run: |
docker compose up --detach --wait dremio dremio-init flightsql-sqlite-test postgres-test
pytest -vvs docs/source/tests/