Skip to content

Implement initial 'gdal raster pipeline' and 'gdal raster reproject' #8322

Implement initial 'gdal raster pipeline' and 'gdal raster reproject'

Implement initial 'gdal raster pipeline' and 'gdal raster reproject' #8322

Workflow file for this run

name: Linux Builds
on:
push:
paths-ignore:
- 'doc/**'
- 'docker/**'
branches-ignore:
- 'backport**'
- 'dependabot**'
pull_request:
paths-ignore:
- 'doc/**'
- 'docker/**'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
linux-build:
# Store the components of the container name as environment variables:
# ${CONTAINER_REGISTRY}/${CONTAINER_REGISTRY_USER}/${CONTAINER_NAME}:${CONTAINER_TAG}
#
# Additionally, CACHE_CONTAINER_TAG may be used as as source for the
# Docker build cache. So if the Dockerfile in a feature branch is
# unchanged relative to master, a full container rebuild should not
# be required.
env:
CONTAINER_REGISTRY: ${{ vars.gdal_container_registry || 'ghcr.io' }}
CONTAINER_REGISTRY_USER: ${{ vars.gdal_container_registry_user || github.repository_owner }}
CONTAINER_NAME: gdal-deps
CONTAINER_TAG: ${{ matrix.container }}-${{ github.base_ref || github.ref_name }}
CACHE_CONTAINER_TAG: ${{ matrix.container }}-master
permissions:
packages: write
strategy:
fail-fast: false
matrix:
# Matrix variables:
#
# * name : readable summary of configuration, used for display
# * id : used as a ccache key, and to create a build subdirectory
# * container : build environment container and path to build script
# * use_avx2 : if true, determine arch at runtime and use in ccache key
# * build_script : name of custom build script, if any. Will be executed
# inside container, from build subdirectory.
# * before_test_script : name of script to run before tests, if any.
# Will be executed outside container, from
# workspace root. Can be used to start docker
# containers as services for testing.
# * test_script : name of custom test script, if any. Will be executed
# inside container, from build subdirectory.
# * travis_branch : value of TRAVIS_BRANCH environment variable,
# used for test skipping
include:
- name: Alpine, gcc
id: alpine
container: alpine
build_script: build.sh
os: ubuntu-22.04
- name: Alpine, clang 32-bit
id: alpine_32bit
container: alpine_32bit
build_script: build.sh
test_script: test.sh
travis_branch: alpine_32bit
os: ubuntu-22.04
- name: Fedora Rawhide, clang++
id: fedora_rawhide
travis_branch: fedora_rawhide
container: fedora_rawhide
build_script: build.sh
os: ubuntu-22.04
- name: Ubuntu 24.04, gcc
id: ubuntu_24.04
travis_branch: ubuntu_2404
container: ubuntu_24.04
before_test_script: services.sh
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 22.04, gcc
id: ubuntu_22.04
travis_branch: ubuntu_2204
container: ubuntu_22.04
before_test_script: services.sh
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 22.04, clang ASAN
id: asan
travis_branch: sanitize
container: ubuntu_22.04
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 20.04, gcc
id: ubuntu_20.04
travis_branch: ubuntu_2004
container: ubuntu_20.04
use_avx2: true
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 20.04, coverage
id: coverage
travis_branch: ubuntu_2004
container: ubuntu_20.04
before_test_script: services.sh
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 20.04, benchmarks
id: benchmarks
travis_branch: ubuntu_2004
container: ubuntu_20.04
build_script: build.sh
test_script: test.sh
os: ubuntu-22.04
- name: Ubuntu 20.04, Intel compiler
id: icc
container: icc
build_script: build.sh
os: ubuntu-22.04
name: ${{ matrix.name }}
runs-on: ${{ matrix.os }}
defaults:
run:
# bash is needed to use ${CONTAINER_REGISTRY_USER,,}, which forces the
# username to lower-case as required by docker.
shell: bash
steps:
- name: Set variables
# This logic needs to be kept in sync between linux_build.yml and converity_scan.yml
run: |
CONTAINER_TAG_CLEAN=$(echo ${CONTAINER_TAG} | tr -d -c "[:alnum:].-")
echo "CONTAINER_TAG_CLEAN=${CONTAINER_TAG_CLEAN}"
echo "CONTAINER_TAG_CLEAN=${CONTAINER_TAG_CLEAN}" >> ${GITHUB_ENV}
CACHE_CONTAINER_TAG_CLEAN=$(echo ${CACHE_CONTAINER_TAG} | tr -d -c "[:alnum:].-")
echo "CACHE_CONTAINER_TAG_CLEAN=${CACHE_CONTAINER_TAG_CLEAN}"
echo "CACHE_CONTAINER_TAG_CLEAN=${CACHE_CONTAINER_TAG_CLEAN}" >> ${GITHUB_ENV}
echo "CONTAINER_NAME_FULL=${CONTAINER_REGISTRY}/${CONTAINER_REGISTRY_USER,,}/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN}" >>${GITHUB_ENV}
# Work around segfaults in ASan/MSan jobs
# Cf https://github.com/libjpeg-turbo/libjpeg-turbo/commit/2dfe6c0fe9e18671105e94f7cbf044d4a1d157e6
# and https://github.com/actions/runner-images/issues/9491
- name: Set up build
run: |
sudo sysctl vm.mmap_rnd_bits=28
- name: Checkout
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- name: Login to Docker Hub
if: env.CONTAINER_REGISTRY == 'docker.io'
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GHCR
if: env.CONTAINER_REGISTRY == 'ghcr.io'
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
# Pull build environment in forks or pull requests, unless [skip cache] is included in the commit message
- name: Pull build environment
if: "(github.repository_owner != 'OSGeo' || github.event_name == 'pull_request') && !contains(github.event.head_commit.message, '[skip cache]')"
run: |
docker pull ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN} || true
docker pull ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CACHE_CONTAINER_TAG_CLEAN} || true
docker pull ${CONTAINER_NAME_FULL} || true
echo "DOCKER_BUILD_CACHE_FROM=--cache-from ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CONTAINER_TAG_CLEAN} --cache-from ${CONTAINER_REGISTRY}/osgeo/${CONTAINER_NAME}:${CACHE_CONTAINER_TAG_CLEAN} --cache-from ${CONTAINER_NAME_FULL}" >>${GITHUB_ENV}
- name: Prepare build context
run: |
mkdir docker-build-context
cp autotest/requirements.txt docker-build-context
- name: Update build environment
env:
DOCKER_BUILDKIT: 1
run: |
# FIXME: for some reason, the fedora rawhide container pushed by
# master job is corrupted (looks like it contains an outdated layer
# symlinking libssl.so.3 to an older version of the actual file),
# once it is pushed. But in the job that generates it,
# compilation & tests work fine. It looks like some weird caching
# issue
if test "${{ matrix.container }}" = "fedora_rawhide"; then
DOCKER_BUILD_CACHE_FROM=""
else
BUILD_ARG_INLINE_CACHE="--build-arg BUILDKIT_INLINE_CACHE=1"
fi
docker build \
${BUILD_ARG_INLINE_CACHE} \
${DOCKER_BUILD_CACHE_FROM} \
-t ${CONTAINER_NAME_FULL} \
-f .github/workflows/${{ matrix.container }}/Dockerfile.ci \
docker-build-context
# Get the architecture so we can use it as part of the cache key,
# but only if we are going to use avx2 in the build. If we are not,
# including the arch will cause unnecessary cache misses.
- name: Get Architecture
id: get-arch
if: matrix.use_avx2
run: |
export ARCH=$(cc -march=native -### -E - < /dev/null 2>&1 | sed -ne 's/.*cc1 .*-march=\([^ "]*\)[ "].*/\1/p')
echo "Architecture: $ARCH"
echo "arch=$ARCH" >> $GITHUB_OUTPUT
# cache the .ccache directory
# key it on the runner os, build type, deps, and arch
# It's especially important to include arch in the key because we
# may get runtime errors with -mavx2 from objects built on a
# different architecture.
- name: Restore build cache
id: restore-cache
uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
path: ${{ github.workspace }}/.ccache
key: ${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}-${{ github.run_id }}
restore-keys: |
${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}-${{ github.ref_name }}
${{ matrix.id }}-${{ steps.get-arch.outputs.arch }}
- name: Prepare ccache
run: |
mkdir -p ${{ github.workspace }}/.ccache
chmod -R a+rw ${{ github.workspace }}/.ccache
docker run --rm \
-v ${{ github.workspace }}/.ccache:/.ccache \
-u $(id -u ${USER}):$(id -g ${USER}) \
${CONTAINER_NAME_FULL} \
sh -c "ccache -M 1G && ccache -sp && ccache -z"
# FIXME the default BUILD_CMD here isn't working...we get an error
# about the quotes not matching.
- name: Build
env:
TRAVIS: yes
TRAVIS_BRANCH: ${{ matrix.travis_branch }}
BUILD_NAME: ${{ matrix.travis_branch }}
run: |
if test -f ".github/workflows/${{ matrix.id }}/${{ matrix.build_script }}"; then
BUILD_CMD="$(pwd)/.github/workflows/${{ matrix.id }}/${{ matrix.build_script }}"
else
BUILD_CMD="sh -c 'cmake .. && make -j$(nproc)'"
fi
# For cache
mkdir -p .gdal
mkdir -p build-${{ matrix.id }}
docker run --name gdal-build \
--rm \
-e CI \
-e GITHUB_WORKFLOW \
-e TRAVIS \
-e TRAVIS_BRANCH \
-e BUILD_NAME \
-e "GDAL_SOURCE_DIR=$(pwd)" \
-u $(id -u ${USER}):$(id -g ${USER}) \
-v $(pwd)/.gdal:/.gdal:rw \
-v $(pwd):$(pwd):rw \
-v ${{ github.workspace }}/.ccache:/.ccache:rw \
--workdir $(pwd)/build-${{ matrix.id }} \
${CONTAINER_NAME_FULL} \
${BUILD_CMD}
- name: Summarize ccache
run: |
docker run --rm \
-v ${{ github.workspace }}/.ccache:/.ccache \
-u $(id -u ${USER}):$(id -g ${USER}) \
${CONTAINER_NAME_FULL} \
ccache -s
- name: Save build cache
uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
path: ${{ github.workspace }}/.ccache
key: ${{ steps.restore-cache.outputs.cache-primary-key }}
- name: Start test services
if: matrix.before_test_script
run: |
.github/workflows/${{ matrix.id }}/${{ matrix.before_test_script }}
# --security-opt seccomp=unconfined, so that the userfaulfd syscall is availabledocker run \
- name: Run tests
env:
TRAVIS: yes
TRAVIS_BRANCH: ${{ matrix.travis_branch }}
BUILD_NAME: ${{ matrix.travis_branch }}
run: |
if test -f ".github/workflows/${{ matrix.id }}/${{ matrix.test_script }}"; then
TEST_CMD="$(pwd)/.github/workflows/${{ matrix.id }}/${{ matrix.test_script }}"
else
TEST_CMD="ctest -V -j $(nproc)"
fi
if test "${{ matrix.id }}" = "benchmarks"; then
if test -f /sys/devices/system/cpu/intel_pstate/no_turbo; then
echo "Disable TurboBoost"
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo
fi
fi
# For cache
mkdir -p .gdal
docker run \
-e CI \
-e GITHUB_WORKFLOW \
-e TRAVIS \
-e TRAVIS_BRANCH \
-e BUILD_NAME \
-e "GDAL_SOURCE_DIR=$(pwd)" \
-u $(id -u ${USER}):$(id -g ${USER}) \
--security-opt seccomp=unconfined \
--add-host=host.docker.internal:host-gateway \
--rm \
-v $(pwd)/.gdal:/.gdal \
-v $(pwd):$(pwd) \
--workdir $(pwd)/build-${{ matrix.id }} \
${CONTAINER_NAME_FULL} \
${TEST_CMD}
- name: Coveralls
uses: coverallsapp/github-action@cfd0633edbd2411b532b808ba7a8b5e04f76d2c8 # v2.3.4
if: ${{ matrix.id == 'coverage' }}
with:
format: lcov
file: build-coverage/gdal_filtered.info
- name: Push build environment
if: github.event_name == 'push'
continue-on-error: true
env:
DOCKER_BUILDKIT: 1
run: |
docker push ${CONTAINER_NAME_FULL}
- name: Upload coverage artifacts
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
if: ${{ matrix.id == 'coverage' }}
with:
name: coverage_index.html
path: build-${{ matrix.id }}/coverage_html/index.html
- name: Upload coverage artifacts
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
if: ${{ matrix.id == 'coverage' }}
with:
name: HTML
path: build-${{ matrix.id }}/coverage_html/*
- name: Deploy ssh key (for coverage)
if: ${{ matrix.id == 'coverage' && github.event_name == 'push' && github.ref_name == 'master' && github.repository == 'OSGeo/gdal' }}
shell: bash -l {0}
run: |
mkdir $HOME/.ssh && echo "${{ secrets.GDAL_TEST_COVERAGE_RESULTS_SSH_KEY }}" > $HOME/.ssh/id_rsa
chmod 700 $HOME/.ssh && chmod 600 $HOME/.ssh/id_rsa
ssh-keyscan -t rsa github.com >> $HOME/.ssh/known_hosts
eval `ssh-agent -s`
ssh-add $HOME/.ssh/id_rsa
- name: Deploy to https://github.com/OSGeo/gdal-test-coverage-results
if: ${{ matrix.id == 'coverage' && github.event_name == 'push' && github.ref_name == 'master' && github.repository == 'OSGeo/gdal' }}
shell: bash -l {0}
run: |
set -x
set -e
mkdir -p output_html/coverage_html
cp -r $GITHUB_WORKSPACE/build-${{ matrix.id }}/coverage_html/* output_html/coverage_html/
cd output_html
git init
git config user.email "gdal-test-coverage-results-bot@example.com"
git config user.name "GDAL test coverage results bot"
git remote add origin git@github.com:gdalautotest-coverage-results/gdalautotest-coverage-results.github.io
git remote -v
echo "Results of coverage of GDAL autotest See https://gdalautotest-coverage-results.github.io/coverage_html/index.html" > README.md
git add -A
git commit -m "Update with OSGeo/gdal commit $GITHUB_SHA"
git push -f origin master