Automatically run release tests on liboqs release candidates

.github/workflows/release.yml

@@ -0,0 +1,58 @@
+name: Release tests
+
+on:
+  repository_dispatch:
+    types: [ "liboqs-release" ]
+
+# To trigger this job, generate a GitHub personal access token and run the following command:
+#
+# curl --request POST \
+#      --header "Accept: application/vnd.github+json" \
+#      --header "Authorization: Bearer YOUR_TOKEN_HERE" \
+#      --header "X-GitHub-Api-Version: 2022-11-28" \
+#      --data '{
+#                "event_type": "liboqs-release",
+#                "client_payload": {
+#                                    "provider_ref": "PROVIDER_BRANCH_OR_TAG_HERE",
+#                                    "liboqs_ref": "LIBOQS_BRANCH_OR_TAG_HERE"
+#                                  }
+#              }' \
+#      https://api.github.com/repos/open-quantum-safe/oqs-provider/dispatches
+
+jobs:
+  release-test:
+    runs-on: ubuntu-latest
+    container:
+      image: openquantumsafe/ci-ubuntu-jammy:latest
+
+    steps:
+      - name: Check if requested ref exists
+        env:
+          provider_ref: ${{ github.event.client_payload.provider_ref }}
+        run: |
+          # try both branch and tag
+          wget --quiet \
+               --header "Accept: application/vnd.github+json" \
+               --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+               --header "X-GitHub-Api-Version: 2022-11-28" \
+               https://api.github.com/repos/open-quantum-safe/oqs-provider/branches/$provider_ref || \
+          wget --quiet \
+               --header "Accept: application/vnd.github+json" \
+               --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+               --header "X-GitHub-Api-Version: 2022-11-28" \
+               https://api.github.com/repos/open-quantum-safe/oqs-provider/git/ref/tags/$provider_ref \
+            && echo "provider_ref=$provider_ref" >> "$GITHUB_ENV" \
+            || echo "provider_ref=main" >> "$GITHUB_ENV"
+      - name: Checkout oqs-provider on requested ref if it exists; otherwise, fall back to main
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.provider_ref }}
+      # This is designed to be triggered automatically from liboqs CI, so don't bother validating the liboqs ref.
+      - name: Checkout liboqs at requested ref
+        uses: actions/checkout@v4
+        with:
+          repository: open-quantum-safe/liboqs
+          path: liboqs
+          ref: ${{ github.event.client_payload.liboqs_ref }}
+      - name: Run release tests
+        run: OPENSSL_BRANCH=master ./scripts/release-test-ci.sh

scripts/release-test-ci.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Stop in case of error
+set -e
+
+# To be run as part of a release test only on Linux
+# requires python, pytest, xdist; install e.g. via
+# sudo apt install python3 python3-pytest python3-pytest-xdist python3-psutil
+
+# must be run in main folder
+# multicore machine recommended for fast execution
+
+# expect (ideally latest/release-test) liboqs to be already build and present
+if [ -d liboqs ]; then
+   export LIBOQS_SRC_DIR=`pwd`/liboqs
+else
+   echo "liboqs not found. Exiting."
+   exit 1
+fi
+
+if [ -d oqs-template ]; then
+    # Activate all algorithms
+    sed -i "s/enable\: false/enable\: true/g" oqs-template/generate.yml
+    python3 oqs-template/generate.py
+    ./scripts/fullbuild.sh
+    ./scripts/runtests.sh
+    if [ -f .local/bin/openssl ]; then
+        OPENSSL_MODULES=`pwd`/_build/lib OPENSSL_CONF=`pwd`/scripts/openssl-ca.cnf python3 -m pytest --numprocesses=auto scripts/test_tls_full.py
+    else
+        echo "For full TLS PQ SIG/KEM matrix test, build (latest) openssl locally."
+    fi
+else
+    echo "$0 must be run in main oqs-provider folder. Exiting."
+    exit 1
+fi
+

scripts/release-test.sh

@@ -3,37 +3,35 @@
 # Stop in case of error
 set -e
 
-# To be run as part of a release test only on Linux
-# requires python, pytest, xdist; install e.g. via
-# sudo apt install python3 python3-pytest python3-pytest-xdist python3-psutil
+# Wrapper around the release-test-ci.sh script to preserve uncommitted modifications.
 
-# must be run in main folder
-# multicore machine recommended for fast execution
+# back up git status and checkout a fresh branch with identical staged/unstaged changes
+save_local_git() {
+    # git stash does not have an --allow-empty option, so make sure we have something to stash.
+    # This allows us to safely call git stash pop.
+    tmpfile=$(mktemp ./XXXXXX)
+    git add $tmpfile
+    # back up uncommitted changes
+    git stash push --quiet
+    # restore changes but save stash
+    git stash apply --quiet
+    # delete dummy file
+    git rm -f $tmpfile --quiet
+    # save working branch name
+    working_branch=$(git branch --show-current)
+    # checkout a fresh branch
+    reltest_branch="reltest-$RANDOM"
+    git checkout -b $reltest_branch --quiet
+}
 
-# expect (ideally latest/release-test) liboqs to be already build and present
-if [ -d liboqs ]; then
-   export LIBOQS_SRC_DIR=`pwd`/liboqs
-else
-   echo "liboqs not found. Exiting."
-   exit 1
-fi
-
-if [ -d oqs-template ]; then
-    # just a temp setup
-    git checkout -b reltest
-    # Activate all algorithms
-    sed -i "s/enable\: false/enable\: true/g" oqs-template/generate.yml
-    python3 oqs-template/generate.py
-    rm -rf _build
-    ./scripts/fullbuild.sh
-    ./scripts/runtests.sh
-    if [ -f .local/bin/openssl ]; then
-        OPENSSL_MODULES=`pwd`/_build/lib OPENSSL_CONF=`pwd`/scripts/openssl-ca.cnf python3 -m pytest --numprocesses=auto scripts/test_tls_full.py
-    else
-        echo "For full TLS PQ SIG/KEM matrix test, build (latest) openssl locally."
-    fi
-    git reset --hard && git checkout main && git branch -D reltest
-else
-    echo "$0 must be run in main oqs-provider folder. Exiting."
-fi
+# restore git status
+restore_local_git() {
+    # switch back to working branch; delete temporary branch; reset to HEAD; pop stashed changes; delete dummy file
+    git switch $working_branch --quiet && git branch -D $reltest_branch --quiet && git reset --hard --quiet && git stash pop --quiet && git rm -f $tmpfile --quiet
+}
 
+save_local_git
+trap restore_local_git EXIT
+# clean out the build directory and run tests
+rm -rf _build
+./scripts/release-test-ci.sh