From c2e780e2d3351bc81264230cd652e239b7aad6c5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 11 Sep 2023 11:05:15 +0900 Subject: [PATCH] Separate verify RC CI jobs from packages (#99) Closes GH-98 --- .github/workflows/package.yaml | 104 ------------------------------- .github/workflows/test.yaml | 56 ++++------------- .github/workflows/verify-rc.yaml | 79 +++++++++++++++++++++++ ci/scripts/prepare-macos.sh | 30 +++++++++ ci/scripts/prepare-ubuntu.sh | 54 ++++++++++++++++ dev/release/verify-rc.sh | 1 + 6 files changed, 175 insertions(+), 149 deletions(-) create mode 100644 .github/workflows/verify-rc.yaml create mode 100755 ci/scripts/prepare-macos.sh create mode 100755 ci/scripts/prepare-ubuntu.sh diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml index bbfa489..036984c 100644 --- a/.github/workflows/package.yaml +++ b/.github/workflows/package.yaml @@ -187,107 +187,3 @@ jobs: local \ package/${PACKAGE}/${TASK_NAMESPACE}/repositories popd - - verify-source: - name: Verify source on ${{ matrix.os }} - needs: - - source - runs-on: ${{ matrix.os }}-latest - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - os: - - macos - - ubuntu - postgresql-version: - - "15" - env: - BUNDLE_GEMFILE: ${{ github.workspace }}/apache-arrow-flight-sql-postgresql/Gemfile - steps: - - uses: actions/download-artifact@v3 - with: - name: source - - name: Extract source archive - run: | - tar_gz=$(echo apache-arrow-flight-sql-postgresql-*.tar.gz) - tar xf ${tar_gz} - ln -s ${tar_gz%.tar.gz} apache-arrow-flight-sql-postgresql - - name: Install dependencies on Ubuntu - if: matrix.os == 'ubuntu' - run: | - sudo apt update - sudo apt install -y -V \ - ca-certificates \ - gpg \ - lsb-release \ - wget - os=$(lsb_release --id --short | tr 'A-Z' 'a-z') - code_name=$(lsb_release --codename --short) - - apt_source_deb=apache-arrow-apt-source-latest-${code_name}.deb - wget https://apache.jfrog.io/artifactory/arrow/${os}/${apt_source_deb} - sudo apt install -y -V ./${apt_source_deb} - - wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ - sudo gpg --no-default-keyring --keyring /usr/share/keyrings/pgdg.gpg --import - - (echo "Types: deb"; \ - echo "URIs: http://apt.postgresql.org/pub/repos/apt"; \ - echo "Suites: ${code_name}-pgdg"; \ - echo "Components: main"; \ - echo "Signed-By: /usr/share/keyrings/pgdg.gpg") | \ - sudo tee /etc/apt/sources.list.d/pgdg.sources - - sudo apt update - sudo apt -y -V purge '^postgresql' - sudo apt install -y -V \ - libarrow-flight-sql-glib-dev \ - libkrb5-dev \ - meson \ - ninja-build \ - postgresql-${{ matrix.postgresql-version }} \ - postgresql-server-dev-${{ matrix.postgresql-version }} - - name: Install dependencies on macOS - if: matrix.os == 'macos' - run: | - pushd apache-arrow-flight-sql-postgresql - sed \ - -i '' \ - -e 's/postgresql@[0-9]*/postgresql@${{ matrix.postgresql-version }}/g' \ - Brewfile - brew update - brew bundle - echo "$(brew --prefix postgresql@${{ matrix.postgresql-version }})/bin" >> $GITHUB_PATH - popd - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ruby - bundler-cache: true - cache-version: 1 - - name: Run - run: | - pushd apache-arrow-flight-sql-postgresql - bundle exec dev/release/verify-rc.sh - popd - env: - TEST_DEFAULT: "0" - TEST_SOURCE: "1" - - verify-binaries: - name: Verify binaries - if: >- - github.ref_type == 'tag' - needs: - - linux - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - name: Run - run: | - version=${GITHUB_REF_NAME%-rc*} - rc=${GITHUB_REF_NAME#*-rc} - dev/release/verify-rc.sh ${version} ${rc} - env: - TEST_BINARIES: "1" - TEST_DEFAULT: "0" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eebb05d..35477b6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -43,50 +43,9 @@ jobs: - "15" steps: - uses: actions/checkout@v4 - - name: Install dependencies on Ubuntu - if: matrix.os == 'ubuntu' + - name: Prepare run: | - sudo apt update - sudo apt install -y -V \ - ca-certificates \ - gpg \ - lsb-release \ - wget - os=$(lsb_release --id --short | tr 'A-Z' 'a-z') - code_name=$(lsb_release --codename --short) - - apt_source_deb=apache-arrow-apt-source-latest-${code_name}.deb - wget https://apache.jfrog.io/artifactory/arrow/${os}/${apt_source_deb} - sudo apt install -y -V ./${apt_source_deb} - - wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ - sudo gpg --no-default-keyring --keyring /usr/share/keyrings/pgdg.gpg --import - - (echo "Types: deb"; \ - echo "URIs: http://apt.postgresql.org/pub/repos/apt"; \ - echo "Suites: ${code_name}-pgdg"; \ - echo "Components: main"; \ - echo "Signed-By: /usr/share/keyrings/pgdg.gpg") | \ - sudo tee /etc/apt/sources.list.d/pgdg.sources - - sudo apt update - sudo apt -y -V purge '^postgresql' - sudo apt install -y -V \ - libarrow-flight-sql-glib-dev \ - libkrb5-dev \ - meson \ - ninja-build \ - postgresql-${{ matrix.postgresql-version }} \ - postgresql-server-dev-${{ matrix.postgresql-version }} - - name: Install dependencies on macOS - if: matrix.os == 'macos' - run: | - sed \ - -i '' \ - -e 's/postgresql@[0-9]*/postgresql@${{ matrix.postgresql-version }}/g' \ - Brewfile - brew update - brew bundle - echo "$(brew --prefix postgresql@${{ matrix.postgresql-version }})/bin" >> $GITHUB_PATH + ci/scripts/prepare-${{ matrix.os }}.sh ${{ matrix.postgresql-version }} - name: Install Apache Arrow Flight SQL adapter run: | setup_args=( @@ -109,6 +68,13 @@ jobs: ruby-version: ruby bundler-cache: true cache-version: 1 - - name: Test + - name: Run run: | - PATH=$(pg_config --bindir):$PATH bundle exec test/run.rb + ARROW_FLIGHT_SQL_POSTGRESQL_MESON_SETUP_ARGS="-Dbenchmark=true" + if [ ${{ matrix.os }} = "macos" ]; then + ARROW_FLIGHT_SQL_POSTGRESQL_MESON_SETUP_ARGS+=" --pkg-config-path=$(brew --prefix libpq)/lib/pkgconfig" + fi + bundle exec dev/release/verify-rc.sh + env: + TEST_DEFAULT: "0" + TEST_SOURCE: "1" diff --git a/.github/workflows/verify-rc.yaml b/.github/workflows/verify-rc.yaml new file mode 100644 index 0000000..c68f158 --- /dev/null +++ b/.github/workflows/verify-rc.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Verify RC + +on: + push: + branches: + - "!**" + tags: + - "*-rc*" + +permissions: + contents: read + +jobs: + source: + name: Source on ${{ matrix.os }} + runs-on: ${{ matrix.os }}-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: + - macos + - ubuntu + postgresql-version: + - "15" + env: + BUNDLE_GEMFILE: ${{ github.workspace }}/apache-arrow-flight-sql-postgresql/Gemfile + steps: + - uses: actions/checkout@v4 + - name: Prepare + run: | + ci/scripts/prepare-${{ matrix.os }}.sh ${{ matrix.postgresql-version }} + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ruby + bundler-cache: true + cache-version: 1 + - name: Run + run: | + version=${GITHUB_REF_NAME%-rc*} + rc=${GITHUB_REF_NAME#*-rc} + bundle exec dev/release/verify-rc.sh ${version} ${rc} + env: + TEST_DEFAULT: "0" + TEST_SOURCE: "1" + + binaries: + name: Binaries + needs: + - linux + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Run + run: | + version=${GITHUB_REF_NAME%-rc*} + rc=${GITHUB_REF_NAME#*-rc} + dev/release/verify-rc.sh ${version} ${rc} + env: + TEST_BINARIES: "1" + TEST_DEFAULT: "0" diff --git a/ci/scripts/prepare-macos.sh b/ci/scripts/prepare-macos.sh new file mode 100755 index 0000000..671fc14 --- /dev/null +++ b/ci/scripts/prepare-macos.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eux + +postgresql_version=$1 + +sed \ + -i "" \ + -e "s/postgresql@[0-9]*/postgresql@${postgresql_version}/g" \ + Brewfile +brew update +brew bundle +echo "$(brew --prefix postgresql@${postgresql_version})/bin" >> "${GITHUB_PATH}" diff --git a/ci/scripts/prepare-ubuntu.sh b/ci/scripts/prepare-ubuntu.sh new file mode 100755 index 0000000..41770d4 --- /dev/null +++ b/ci/scripts/prepare-ubuntu.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eux + +postgresql_version=$1 + +sudo apt update +sudo apt install -y -V \ + ca-certificates \ + gpg \ + lsb-release \ + wget +os=$(lsb_release --id --short | tr 'A-Z' 'a-z') +code_name=$(lsb_release --codename --short) + +apt_source_deb=apache-arrow-apt-source-latest-${code_name}.deb +wget https://apache.jfrog.io/artifactory/arrow/${os}/${apt_source_deb} +sudo apt install -y -V ./${apt_source_deb} + +wget -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ + sudo gpg --no-default-keyring --keyring /usr/share/keyrings/pgdg.gpg --import - +(echo "Types: deb"; \ + echo "URIs: http://apt.postgresql.org/pub/repos/apt"; \ + echo "Suites: ${code_name}-pgdg"; \ + echo "Components: main"; \ + echo "Signed-By: /usr/share/keyrings/pgdg.gpg") | \ + sudo tee /etc/apt/sources.list.d/pgdg.sources + +sudo apt update +sudo apt -y -V purge '^postgresql' +sudo apt install -y -V \ + libarrow-flight-sql-glib-dev \ + libkrb5-dev \ + meson \ + ninja-build \ + postgresql-${postgresql_version} \ + postgresql-server-dev-${postgresql_version} diff --git a/dev/release/verify-rc.sh b/dev/release/verify-rc.sh index 3998b12..d25a732 100755 --- a/dev/release/verify-rc.sh +++ b/dev/release/verify-rc.sh @@ -205,6 +205,7 @@ test_source_distribution() { meson setup \ --prefix="${PWD}/install" \ -Dpostgresql_dir="$(pg_config --bindir)/.." \ + ${ARROW_FLIGHT_SQL_POSTGRESQL_MESON_SETUP_ARGS:-} \ build meson compile -C build if [ ${TEST_SOURCE_MANUAL} -gt 0 ]; then