diff --git a/.github/workflows/publish-github-release.yml b/.github/workflows/publish-github-release.yml new file mode 100644 index 00000000..322391ff --- /dev/null +++ b/.github/workflows/publish-github-release.yml @@ -0,0 +1,111 @@ +# workflows/publish-github-release.yml +# +# Publish GitHub Release +# Publish the ParadeDB pg_analytics GitHub Release. + +name: Publish GitHub Release + +on: + push: + branches: + - main + - dev + workflow_dispatch: + +concurrency: + group: publish-github-release-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + publish-github-release: + name: Publish ParadeDB pg_analytics GitHub Release + runs-on: depot-ubuntu-latest-2 + + steps: + - name: Checkout Git Repository + uses: actions/checkout@v4 + + - name: Set Environment + id: env + run: | + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "environment=prod" >> $GITHUB_OUTPUT + echo "Using prod configuration..." + else + echo "environment=dev" >> $GITHUB_OUTPUT + echo "Using dev configuration..." + fi + + # We store the GitHub Release version number in GitHub Actions Variables. Since it's + # not possible for a GHA variable to be negative, we store the version of the next + # release, to allow 0-indexing. This is why we immediately release the version stored, + # and increment it after the GitHub release is created. + - name: Retrieve & Increment Release Version Number + id: version + run: | + if [[ "${{ steps.env.outputs.environment }}" == "prod" ]]; then + echo 'Using prod configuration...' + CURRENT_RELEASE_VERSION="${{ vars.VERSION_MAJOR }}.${{ vars.VERSION_MINOR }}.${{ vars.VERSION_PATCH }}" + + # Increment GHA variable version by 0.0.1 for next release + GHA_VAR_NAME="VERSION_PATCH" + GHA_VAR_VALUE="$(( ${{ vars.VERSION_PATCH }} + 1 ))" + elif [[ "${{ steps.env.outputs.environment }}" == "dev" ]]; then + echo 'Using dev configuration...' + CURRENT_RELEASE_VERSION="${{ vars.VERSION_MAJOR }}.${{ vars.VERSION_MINOR }}.${{ vars.VERSION_PATCH }}-dev-rc.${{ vars.VERSION_DEV_RC }}" + + # Increment GHA variable version by dev-rc.1 for next release + GHA_VAR_NAME="VERSION_DEV_RC" + GHA_VAR_VALUE="$(( ${{ vars.VERSION_DEV_RC }} + 1 ))" + else + echo "Error: Invalid branch" && false + fi + + # Output the current release version to create the GitHub Release tag, and the new version to update GitHub Actions variable + echo "version=${CURRENT_RELEASE_VERSION}" >> $GITHUB_OUTPUT + echo "gha_var_name=${GHA_VAR_NAME}" >> $GITHUB_OUTPUT + echo "gha_var_value=${GHA_VAR_VALUE}" >> $GITHUB_OUTPUT + + - name: Update Version Number in GitHub Actions Variables + env: + GH_TOKEN: ${{ secrets.GHA_CREATE_RELEASE_PAT }} + run: | + # on prod we update patch and reset dev RC to 0 + if [[ "${{ steps.env.outputs.environment }}" == "prod" ]]; then + gh api \ + --method PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/paradedb/paradedb/actions/variables/${{ steps.version.outputs.gha_var_name }} \ + -f name='${{ steps.version.outputs.gha_var_name }}' \ + -f value='${{ steps.version.outputs.gha_var_value }}' + + gh api \ + --method PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/paradedb/paradedb/actions/variables/VERSION_DEV_RC \ + -f name='VERSION_DEV_RC' \ + -f value='0' + # on dev we only update dev RC + elif [[ "${{ steps.env.outputs.environment }}" == "dev" ]]; then + gh api \ + --method PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/paradedb/paradedb/actions/variables/${{ steps.version.outputs.gha_var_name }} \ + -f name='${{ steps.version.outputs.gha_var_name }}' \ + -f value='${{ steps.version.outputs.gha_var_value }}' + else + echo "Error: Invalid branch" && false + fi + + # We create the GitHub release last in case of failure in previous steps + - name: Create GitHub Release (prod only) + if: steps.env.outputs.environment == 'prod' + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.version.outputs.version }} + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GHA_CREATE_RELEASE_PAT }} diff --git a/.github/workflows/publish-pg_analytics.yml b/.github/workflows/publish-pg_analytics.yml new file mode 100644 index 00000000..e35ae15a --- /dev/null +++ b/.github/workflows/publish-pg_analytics.yml @@ -0,0 +1,442 @@ +# workflows/publish-pg_analytics.yml +# +# Publish pg_analytics +# Build and publish the pg_analytics extension as .deb and .rpm to GitHub Releases. + +name: Publish pg_analytics + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + version: + description: "The version to set for the pg_analytics release. This publishes the latest commit of the chosen branch and uploads it to the pre-existing GitHub Release of the provided version." + required: true + default: "" + +concurrency: + group: publish-pg_analytics-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +# Used by actions/attest-build-provenance to sign the builds +permissions: + id-token: write + attestations: write + +jobs: + publish-pg_analytics: + name: Publish pg_analytics for PostgreSQL ${{ matrix.pg_version }} on ${{ matrix.image }} ${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + container: + image: ${{ matrix.image }} + strategy: + matrix: + include: + # Ubuntu 22.04 + - runner: depot-ubuntu-latest-8 + image: ubuntu:22.04 + pg_version: 14 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: ubuntu:22.04 + pg_version: 14 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: ubuntu:22.04 + pg_version: 15 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: ubuntu:22.04 + pg_version: 15 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: ubuntu:22.04 + pg_version: 16 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: ubuntu:22.04 + pg_version: 16 + arch: arm64 + # Ubuntu 24.04 + - runner: depot-ubuntu-latest-8 + image: ubuntu:24.04 + pg_version: 14 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: ubuntu:24.04 + pg_version: 14 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: ubuntu:24.04 + pg_version: 15 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: ubuntu:24.04 + pg_version: 15 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: ubuntu:24.04 + pg_version: 16 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: ubuntu:24.04 + pg_version: 16 + arch: arm64 + # Debian 12 + - runner: depot-ubuntu-latest-8 + image: debian:12-slim + pg_version: 14 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: debian:12-slim + pg_version: 14 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: debian:12-slim + pg_version: 15 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: debian:12-slim + pg_version: 15 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: debian:12-slim + pg_version: 16 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: debian:12-slim + pg_version: 16 + arch: arm64 + # Red Hat Enterprise Linux 8 + - runner: depot-ubuntu-latest-8 + image: redhat/ubi8:latest + pg_version: 14 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: redhat/ubi8:latest + pg_version: 14 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: redhat/ubi8:latest + pg_version: 15 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: redhat/ubi8:latest + pg_version: 15 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: redhat/ubi8:latest + pg_version: 16 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: redhat/ubi8:latest + pg_version: 16 + arch: arm64 + # Red Hat Enterprise Linux 9 + - runner: depot-ubuntu-latest-8 + image: redhat/ubi9:latest + pg_version: 14 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: redhat/ubi9:latest + pg_version: 14 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: redhat/ubi9:latest + pg_version: 15 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: redhat/ubi9:latest + pg_version: 15 + arch: arm64 + - runner: depot-ubuntu-latest-8 + image: redhat/ubi9:latest + pg_version: 16 + arch: amd64 + - runner: depot-ubuntu-latest-arm-4 + image: redhat/ubi9:latest + pg_version: 16 + arch: arm64 + + steps: + - name: Checkout Git Repository + uses: actions/checkout@v4 + + # We remove the existing pg_analytics/ directory, which just contains the README.md + - name: Clone paradedb/pg_analytics Repository & Add pg_analytics to Cargo.toml + run: | + rm -rf pg_analytics/ + git clone https://github.com/paradedb/pg_analytics.git pg_analytics/ + sed -i '/members = \[/,/\]/s/\]/ "pg_analytics",\n]/' Cargo.toml + + - name: Install Debian Dependencies + if: ${{ matrix.image == 'debian:11-slim' || matrix.image == 'debian:12-slim' || matrix.image == 'ubuntu:22.04' || matrix.image == 'ubuntu:24.04' }} + run: DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y sudo wget curl gnupg lsb-release pkg-config libssl-dev jq + + - name: Install RHEL Dependencies + if: ${{ matrix.image == 'redhat/ubi8:latest' || matrix.image == 'redhat/ubi9:latest' }} + run: | + # Determine RHEL version to retrieve the correct repositories + if [ "${{ matrix.image }}" == "redhat/ubi8:latest" ]; then + RHEL_VERSION=8 + elif [ "${{ matrix.image }}" == "redhat/ubi9:latest" ]; then + RHEL_VERSION=9 + else + echo "Unsupported RHEL version" + exit 1 + fi + + # Install dependencies + dnf install -y sudo wget gcc llvm-toolset pkgconf-pkg-config openssl-devel jq rpm-build + + # Add Oracle Linux ${RHEL_VERSION} repositories to enable epel-release + sudo tee /etc/yum.repos.d/oracle-linux-ol${RHEL_VERSION}.repo > /dev/null <> $GITHUB_OUTPUT + + if [ -x "$(command -v lsb_release)" ]; then + DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') + RELEASE=$(lsb_release -cs) + else + DISTRO=$(cat /etc/os-release | grep ^ID= | cut -d= -f2 | tr -d '"') + RELEASE=$(cat /etc/os-release | grep ^VERSION_ID= | cut -d= -f2 | tr -d '"' | cut -d. -f1) + fi + if [ "$DISTRO" = "ubuntu" ]; then + OS_VERSION="$RELEASE" + elif [ "$DISTRO" = "debian" ]; then + OS_VERSION="$RELEASE" + elif [ "$DISTRO" = "rhel" ]; then + OS_VERSION="el$RELEASE" + else + OS_VERSION="$DISTRO-$RELEASE" + fi + echo "OS Version: $OS_VERSION" + echo "os_version=$OS_VERSION" >> $GITHUB_OUTPUT + + # RHEL uses different architecture and OS naming conventions when packaging via rpmbuid. We need to + # retrieve those as well to programmatically retrieve and upload the correct RPM package + if [[ "${{ matrix.image }}" == "redhat/ubi8:latest" || "${{ matrix.image }}" == "redhat/ubi9:latest" ]]; then + if [ "${{ matrix.arch }}" == "amd64" ]; then + ARCH=x86_64 + else + ARCH=aarch64 + fi + echo "RPM Arch: $ARCH" + echo "rpm_arch=$ARCH" >> $GITHUB_OUTPUT + fi + + - name: Install & Configure Supported PostgreSQL Version on Debian + if: ${{ matrix.image == 'debian:11-slim' || matrix.image == 'debian:12-slim' || matrix.image == 'ubuntu:22.04' || matrix.image == 'ubuntu:24.04' }} + run: | + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + sudo DEBIAN_FRONTEND=noninteractive apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} debhelper devscripts dput gnupg + sudo chown -R $(whoami) /usr/share/postgresql/${{ matrix.pg_version }}/ /usr/lib/postgresql/${{ matrix.pg_version }}/ /var/lib/postgresql/${{ matrix.pg_version }}/ + echo "/usr/lib/postgresql/${{ matrix.pg_version }}/bin" >> $GITHUB_PATH + + - name: Install & Configure Supported PostgreSQL Version on RHEL + if: ${{ matrix.image == 'redhat/ubi8:latest' || matrix.image == 'redhat/ubi9:latest' }} + run: | + # Determine RHEL version to retrieve the correct repositories + if [ "${{ matrix.image }}" == "redhat/ubi8:latest" ]; then + RHEL_VERSION=8 + elif [ "${{ matrix.image }}" == "redhat/ubi9:latest" ]; then + RHEL_VERSION=9 + else + echo "Unsupported RHEL version" + exit 1 + fi + + # Install the repository RPM: + sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-${RHEL_VERSION}-${{ steps.version.outputs.rpm_arch }}/pgdg-redhat-repo-latest.noarch.rpm + + # Disable the built-in PostgreSQL module: + sudo dnf -qy module disable postgresql + + # Install PostgreSQL: + sudo dnf install -y postgresql${{ matrix.pg_version }} postgresql${{ matrix.pg_version }}-server postgresql${{ matrix.pg_version }}-devel + + - name: Install pgrx + run: cargo install -j $(nproc) --locked cargo-pgrx --version 0.11.3 + + # Note: We need to specify bash as the shell to ensure that it doesn't default to /bin/sh on Debian, which doesn't support the `[[` syntax + - name: Initialize pgrx for Current PostgreSQL Version + shell: bash + run: | + if [[ "${{ matrix.image }}" == "redhat/ubi8:latest" || "${{ matrix.image }}" == "redhat/ubi9:latest" ]]; then + PG_CONFIG_PATH="/usr/pgsql-${{ matrix.pg_version }}/bin/pg_config" + else + PG_CONFIG_PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config" + fi + cargo pgrx init --pg${{ matrix.pg_version }}=$PG_CONFIG_PATH + + - name: Package pg_analytics Extension with pgrx + run: | + if [[ "${{ matrix.image }}" == "redhat/ubi8:latest" || "${{ matrix.image }}" == "redhat/ubi9:latest" ]]; then + PG_CONFIG_PATH="/usr/pgsql-${{ matrix.pg_version }}/bin/pg_config" + else + PG_CONFIG_PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config" + fi + cargo pgrx package --pg-config $PG_CONFIG_PATH + + - name: Create .deb Package + if: ${{ matrix.image == 'debian:11-slim' || matrix.image == 'debian:12-slim' || matrix.image == 'ubuntu:22.04' || matrix.image == 'ubuntu:24.04' }} + run: | + # Create installable package + mkdir archive + cp `find target/release -type f -name "pg_analytics*"` archive + package_dir=pg_analytics-${{ steps.version.outputs.tag_version }}-${{ steps.version.outputs.os_version }}-${{ matrix.arch }}-pg${{ matrix.pg_version }} + + # Copy files into directory structure + mkdir -p ${package_dir}/usr/lib/postgresql/${{ matrix.pg_version }}/lib + mkdir -p ${package_dir}/usr/share/postgresql/${{ matrix.pg_version}}/extension + cp archive/*.so ${package_dir}/usr/lib/postgresql/${{ matrix.pg_version }}/lib + cp archive/*.control ${package_dir}/usr/share/postgresql/${{ matrix.pg_version }}/extension + cp archive/*.sql ${package_dir}/usr/share/postgresql/${{ matrix.pg_version }}/extension + + # Create control file (package name cannot have underscore) + mkdir -p ${package_dir}/DEBIAN + touch ${package_dir}/DEBIAN/control + deb_version=${{ steps.version.outputs.tag_version }} + CONTROL_FILE="${package_dir}/DEBIAN/control" + echo 'Package: postgresql-${{ matrix.pg_version }}-pg-analytics' >> $CONTROL_FILE + echo 'Version:' ${deb_version} >> $CONTROL_FILE + echo 'Section: database' >> $CONTROL_FILE + echo 'Priority: optional' >> $CONTROL_FILE + echo 'Architecture: ${{ matrix.arch }}' >> $CONTROL_FILE + echo 'Depends: postgresql-${{ matrix.pg_version }}' >> $CONTROL_FILE + echo 'Maintainer: ParadeDB ' >> $CONTROL_FILE + echo 'Description: DuckDB-powered analytics for Postgres' >> $CONTROL_FILE + + # Create .deb package + sudo chown -R root:root ${package_dir} + sudo chmod -R 755 ${package_dir} + sudo dpkg-deb -Zxz --build --root-owner-group ${package_dir} + + - name: Create .rpm Package + if: ${{ matrix.image == 'redhat/ubi8:latest' || matrix.image == 'redhat/ubi9:latest' }} + run: | + echo "Configuring RPM build environment..." + rpmdev-setuptree + + echo "Creating RPM SPEC file..." + spec_file=~/rpmbuild/SPECS/pg_analytics.spec + cat < $spec_file + Name: pg_analytics_${{ matrix.pg_version }} + Version: ${{ steps.version.outputs.tag_version }} + Release: 1%{?dist} + Summary: DuckDB-powered analytics for Postgres + License: GNU Affero General Public License v3.0 + URL: https://github.com/paradedb/pg_analytics + + BuildRequires: postgresql${{ matrix.pg_version }}-devel + Requires: postgresql${{ matrix.pg_version }}-server + + %description + pg_analytics is a Postgres extension that enables fast analytics over + data lakes from Postgres. It is built on top of DuckDB, using pgrx. + + %install + %{__rm} -rf %{buildroot} + install -d %{buildroot}/usr/pgsql-${{ matrix.pg_version }}/lib/ + install -d %{buildroot}/usr/pgsql-${{ matrix.pg_version }}/share/extension/ + install -m 755 %{_sourcedir}/pg_analytics_${{ matrix.pg_version }}/usr/pgsql-${{ matrix.pg_version }}/lib/pg_analytics.so %{buildroot}/usr/pgsql-${{ matrix.pg_version }}/lib/ + install -m 644 %{_sourcedir}/pg_analytics_${{ matrix.pg_version }}/usr/pgsql-${{ matrix.pg_version }}/share/extension/pg_analytics*.sql %{buildroot}/usr/pgsql-${{ matrix.pg_version }}/share/extension/ + install -m 644 %{_sourcedir}/pg_analytics_${{ matrix.pg_version }}/usr/pgsql-${{ matrix.pg_version }}/share/extension/pg_analytics.control %{buildroot}/usr/pgsql-${{ matrix.pg_version }}/share/extension/ + + %files + /usr/pgsql-${{ matrix.pg_version }}/lib/pg_analytics.so + /usr/pgsql-${{ matrix.pg_version }}/share/extension/pg_analytics.control + /usr/pgsql-${{ matrix.pg_version }}/share/extension/pg_analytics*sql + + %changelog + * Thu Jun 6 2024 ParadeDB - ${{ steps.version.outputs.tag_version }} + - Initial RPM Release + EOF + + echo "Copying pg_analytics binaries to RPM build directory..." + cp -r target/release/pg_analytics-pg${{ matrix.pg_version }}/ ~/rpmbuild/SOURCES/pg_analytics_${{ matrix.pg_version }} + + echo "Building RPM package..." + rpmbuild --without debuginfo -ba ~/rpmbuild/SPECS/pg_analytics.spec + + - name: Sign and Attest Build Provenance + uses: actions/attest-build-provenance@v1 + with: + subject-path: | + ./pg_analytics-${{ steps.version.outputs.tag_version }}-${{ steps.version.outputs.os_version }}-${{ matrix.arch }}-pg${{ matrix.pg_version }}.deb + ~/rpmbuild/RPMS/${{ steps.version.outputs.rpm_arch }}/pg_analytics_${{ matrix.pg_version }}-${{ steps.version.outputs.tag_version }}-1.${{ steps.version.outputs.os_version }}.${{ steps.version.outputs.rpm_arch }}.rpm + + - name: Retrieve GitHub Release Upload URL + id: upload_url + run: echo UPLOAD_URL=$(curl --silent https://api.github.com/repos/${{ github.repository }}/releases/tags/v${{ steps.version.outputs.tag_version }} | jq .upload_url --raw-output) >> $GITHUB_OUTPUT + + - name: Upload pg_analytics .deb to GitHub Release + if: ${{ matrix.image == 'debian:11-slim' || matrix.image == 'debian:12-slim' || matrix.image == 'ubuntu:22.04' || matrix.image == 'ubuntu:24.04' }} + uses: shogo82148/actions-upload-release-asset@v1 + with: + github_token: ${{ secrets.GHA_CREATE_RELEASE_PAT }} + upload_url: ${{ steps.upload_url.outputs.upload_url }} + asset_path: ./pg_analytics-${{ steps.version.outputs.tag_version }}-${{ steps.version.outputs.os_version }}-${{ matrix.arch }}-pg${{ matrix.pg_version }}.deb + asset_name: postgresql-${{ matrix.pg_version }}-pg-analytics_${{ steps.version.outputs.tag_version }}-1PARADEDB-${{ steps.version.outputs.os_version }}_${{ matrix.arch }}.deb + overwrite: true + + - name: Upload pg_analytics .rpm to GitHub Release + if: ${{ matrix.image == 'redhat/ubi8:latest' || matrix.image == 'redhat/ubi9:latest' }} + uses: shogo82148/actions-upload-release-asset@v1 + with: + github_token: ${{ secrets.GHA_CREATE_RELEASE_PAT }} + upload_url: ${{ steps.upload_url.outputs.upload_url }} + asset_path: ~/rpmbuild/RPMS/${{ steps.version.outputs.rpm_arch }}/pg_analytics_${{ matrix.pg_version }}-${{ steps.version.outputs.tag_version }}-1.${{ steps.version.outputs.os_version }}.${{ steps.version.outputs.rpm_arch }}.rpm + asset_name: pg_analytics_${{ matrix.pg_version }}-${{ steps.version.outputs.tag_version }}-1PARADEDB.${{ steps.version.outputs.os_version }}.${{ steps.version.outputs.rpm_arch }}.rpm + overwrite: true