diff --git a/.dockerignore b/.dockerignore index ddedce4..c1bb1f6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,7 @@ derivatives/** # ignore documentation docs/** + +# ignore CI scripts +ci/** +.github/** diff --git a/.github/interop/README.md b/.github/interop/README.md deleted file mode 100644 index c46c4e6..0000000 --- a/.github/interop/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Legacy Interop -The scripts here are intended to patch older versions of ldmx-sw -so that they can be built by newer container images. They are -run before the build configuration (`cmake`) step and are run -from the ldmx-sw parent directory. diff --git a/.github/interop/force-legacy-onnx.sh b/.github/interop/force-legacy-onnx.sh deleted file mode 100755 index 2ac61dd..0000000 --- a/.github/interop/force-legacy-onnx.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -# don't have cmake even check for another onnx -sed -i '68,72 {s|^|#|}' ldmx-sw/cmake/FindONNXRuntime.cmake diff --git a/.github/interop/no-testing.sh b/.github/interop/no-testing.sh deleted file mode 100755 index 2abb261..0000000 --- a/.github/interop/no-testing.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -# don't build ldmx-sw tests by commenting out the build_test call -sed -i 's|build_test()|#build_test()|' ldmx-sw/CMakeLists.txt diff --git a/.github/interop/v3.0.0/force-legacy-onnx.sh b/.github/interop/v3.0.0/force-legacy-onnx.sh deleted file mode 120000 index bd75c62..0000000 --- a/.github/interop/v3.0.0/force-legacy-onnx.sh +++ /dev/null @@ -1 +0,0 @@ -../force-legacy-onnx.sh \ No newline at end of file diff --git a/.github/interop/v3.0.0/no-testing.sh b/.github/interop/v3.0.0/no-testing.sh deleted file mode 120000 index dad5644..0000000 --- a/.github/interop/v3.0.0/no-testing.sh +++ /dev/null @@ -1 +0,0 @@ -../no-testing.sh \ No newline at end of file diff --git a/.github/interop/v3.0.2/force-legacy-onnx.sh b/.github/interop/v3.0.2/force-legacy-onnx.sh deleted file mode 120000 index bd75c62..0000000 --- a/.github/interop/v3.0.2/force-legacy-onnx.sh +++ /dev/null @@ -1 +0,0 @@ -../force-legacy-onnx.sh \ No newline at end of file diff --git a/.github/interop/v3.0.2/no-testing.sh b/.github/interop/v3.0.2/no-testing.sh deleted file mode 120000 index dad5644..0000000 --- a/.github/interop/v3.0.2/no-testing.sh +++ /dev/null @@ -1 +0,0 @@ -../no-testing.sh \ No newline at end of file diff --git a/.github/interop/v3.1.13/force-legacy-onnx.sh b/.github/interop/v3.1.13/force-legacy-onnx.sh deleted file mode 120000 index bd75c62..0000000 --- a/.github/interop/v3.1.13/force-legacy-onnx.sh +++ /dev/null @@ -1 +0,0 @@ -../force-legacy-onnx.sh \ No newline at end of file diff --git a/.github/interop/v3.1.13/no-testing.sh b/.github/interop/v3.1.13/no-testing.sh deleted file mode 120000 index dad5644..0000000 --- a/.github/interop/v3.1.13/no-testing.sh +++ /dev/null @@ -1 +0,0 @@ -../no-testing.sh \ No newline at end of file diff --git a/.github/interop/v3.2.4/force-legacy-onnx.sh b/.github/interop/v3.2.4/force-legacy-onnx.sh deleted file mode 120000 index bd75c62..0000000 --- a/.github/interop/v3.2.4/force-legacy-onnx.sh +++ /dev/null @@ -1 +0,0 @@ -../force-legacy-onnx.sh \ No newline at end of file diff --git a/.github/interop/v3.2.4/no-testing.sh b/.github/interop/v3.2.4/no-testing.sh deleted file mode 120000 index dad5644..0000000 --- a/.github/interop/v3.2.4/no-testing.sh +++ /dev/null @@ -1 +0,0 @@ -../no-testing.sh \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1049c81..85b8e87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,24 +123,29 @@ jobs: name: Inspect image run: | docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} + + list-ldmx-sw-to-test: + name: List ldmx-sw Versions to Test + runs-on: ubuntu-latest + outputs: + matrix: ${{steps.read_json.outputs.matrix}} + steps: + - uses: actions/checkout@v4 + - id: read_json + run: echo "matrix=$(jq -c . ci/ldmx-sw-to-test.json)" >> $GITHUB_OUTPUT test: - needs: merge + needs: + - list-ldmx-sw-to-test + - merge runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - ldmx_sw_branch: [ trunk, v3.0.0, v3.0.2, v3.1.13, v3.2.4, v3.2.12, v3.3.0, v3.4.0, v4.0.0 ] + matrix: ${{fromJSON(needs.list-ldmx-sw-to-test.outputs.matrix)}} defaults: run: shell: bash steps: - - - name: Setup QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - name: Download Build Context for Test Script uses: actions/checkout@v4 @@ -156,48 +161,10 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} - - name: Pull down ldmx-sw for testing - uses: actions/checkout@v4 + name: Install just + uses: extractions/setup-just@v2 with: - repository: LDMX-Software/ldmx-sw - submodules: recursive - path: ldmx-sw - ref: ${{ matrix.ldmx_sw_branch }} - fetch-depth: 0 + just-version: 1.26.0 - name: Test the Image - run: | - echo "::group::Init Environment" - # extract a single tag to run tests with - tag=$(jq -r .tags[0] <<<"${DOCKER_METADATA_OUTPUT_JSON}") - export LDMX_BASE=$(pwd) - export _docker_parameters="-i -e LDMX_BASE -v $LDMX_BASE:$LDMX_BASE ${tag}" - mkdir ldmx-sw/build - interop_patches=".github/interop/${{ matrix.ldmx_sw_branch }}" - echo "::endgroup::" - if [ -d ${interop_patches} ]; then - echo "::group::Found ldmx-sw patches for ${{ matrix.ldmx_sw_branch }}" - for patch in ${interop_patches}/*; do - echo "running ${patch}" - ./${patch} - done - echo "::endgroup::" - else - echo "::group::No patches for ${{ matrix.ldmx_sw_branch }}" - echo "::endgroup::" - fi - echo "::group::Build ldmx-sw" - docker run $_docker_parameters $(pwd)/ldmx-sw/build 'cmake .. && make install' - echo "::endgroup::" - echo "::group::Run python test script" - docker run $_docker_parameters $(pwd) python3 test.py - echo "::endgroup::" - if [ ! -d ${interop_patches} ]; then - echo "::group::ldmx-sw test program" - docker run $_docker_parameters $(pwd)/ldmx-sw/build ctest --verbose - echo "::endgroup::" - echo "::group::run a basic inclusive sim" - docker run $_docker_parameters $(pwd) \ - 'LDMX_RUN_NUMBER=1 LDMX_NUM_EVENTS=10 fire ldmx-sw/.github/validation_samples/inclusive/config.py' - echo "::endgroup::" - fi + run: ./ci/test-ldmx-sw $(jq -r .tags[0] <<< "${DOCKER_METADATA_OUTPUT_JSON}") ${{ matrix.ldmx_sw }} diff --git a/.github/workflows/ldmx-sw-dependabot.yml b/.github/workflows/ldmx-sw-dependabot.yml new file mode 100644 index 0000000..7a8a30a --- /dev/null +++ b/.github/workflows/ldmx-sw-dependabot.yml @@ -0,0 +1,36 @@ +--- + +name: Test ldmx-sw Dependabot + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * 0' # every Sunday at midnight + +jobs: + check-on-runner-releases: + permissions: + pull-requests: write + contents: write + actions: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: fetch latest releases + run: | + ./ci/ldmx-sw-dependabot --apply + git config --global user.name ldmx-sw-test-bot + git config --global user.email ldmx-sw-test-bot@users.noreply.github.com + git switch -c auto-ldmx-sw-test-update + git add ci/ldmx-sw-to-test.json + git commit -m "Auto ldmx-sw Version Update" || exit 0 + git push -fu origin auto-runner-update + gh pr create \ + --base main \ + --head auto-runner-update \ + --title "Auto ldmx-sw Version Update" \ + --body "Written by custom dependabot script that checks for new updates. PR now free to be modified/closed." + env: + GH_TOKEN: ${{ github.token }} diff --git a/ci/interop/README.md b/ci/interop/README.md new file mode 100644 index 0000000..d839ec8 --- /dev/null +++ b/ci/interop/README.md @@ -0,0 +1,20 @@ +# Legacy Interop + +The patch files here are intended to patch older versions of ldmx-sw +so that they can be built with newer images that have newer dependencies +and compilers. + +They are `git apply`ed within ldmx-sw and are applied before the configuration (`cmake`) +step so that they can modify the build configuration files if need be. + +For creating a patch files, there is a small script in this directory +that runs the appropriate `git` commands for you. +``` +# inside of the ldmx-sw you have patched +path/to/ci/interop/save-patch +``` + +Many versions of ldmx-sw require the same patch and so instead of copying the +same file, I have just symlinked a specific version's patch file to the previous +version so that developers only need to update a patch file for the version +where the (now breaking) change was introduced. diff --git a/ci/interop/save-patch b/ci/interop/save-patch new file mode 100755 index 0000000..111f3e3 --- /dev/null +++ b/ci/interop/save-patch @@ -0,0 +1,67 @@ +#!/bin/bash +# save the patched version of ldmx-sw into a patch file + +set -o nounset +set -o errexit + +usage() { + cat <<\HELP + + USAGE: + ./path/to/ci/interop/save-patch [path/to/ldmx-sw] [-h|--help] + + ARGUMENTS + -h|--help : print this help and exit + path/to/ldmx-sw : provide full path to ldmx-sw, otherwise tries to use current directory + +HELP +} + +ci_interop_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)" +ldmx_sw_dir="${PWD}" # default to current directory +while [ "$#" -gt 0 ]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -*) + echo "ERROR: Unknown option '${1}'" + usage + exit 1 + ;; + *) + if [ -d "${1}" ]; then + ldmx_sw_dir="$(cd -- "${1}" &> /dev/null && pwd)" + else + echo "ERROR: '${1}' is not a directory." + exit 1 + fi + ;; + esac + shift +done + +cd "${ldmx_sw_dir}" + +if ! git rev-parse --git-dir &> /dev/null; then + echo "ERROR: '${ldmx_sw_dir}' is not a git repository." + exit 2 +fi + +first_remote_url=$(git remote get-url $(git remote)) +if ! [[ "${first_remote_url}" =~ .*LDMX-Software/ldmx-sw.* ]]; then + echo "ERROR: '${ldmx_sw_dir}' is not ldmx-sw. It points to '${first_remote_url}'." + exit 2 +fi + +# inside ldmx-sw +version=$(git describe --tags) +diff_cmd="git diff" +if [[ "${version}" =~ ^v3 ]]; then + diff_cmd="${diff_cmd} --submodule=diff" +fi + +rm ${ci_interop_dir}/${version}.patch +${diff_cmd} > ${ci_interop_dir}/${version}.patch + diff --git a/ci/interop/v3.3.0.patch b/ci/interop/v3.3.0.patch new file mode 100644 index 0000000..8737514 --- /dev/null +++ b/ci/interop/v3.3.0.patch @@ -0,0 +1,13 @@ +Submodule Conditions contains modified content +diff --git a/Conditions/include/Conditions/SimpleTableCondition.h b/Conditions/include/Conditions/SimpleTableCondition.h +index e418c42..79ed3a3 100644 +--- a/Conditions/include/Conditions/SimpleTableCondition.h ++++ b/Conditions/include/Conditions/SimpleTableCondition.h +@@ -7,6 +7,7 @@ + #define FRAMEWORK_SIMPLETABLECONDITION_H_ + + #include ++#include + + #include "Framework/ConditionsObject.h" + #include "Framework/Exception/Exception.h" diff --git a/ci/interop/v3.4.0.patch b/ci/interop/v3.4.0.patch new file mode 100644 index 0000000..de68c03 --- /dev/null +++ b/ci/interop/v3.4.0.patch @@ -0,0 +1,13 @@ +Submodule Conditions contains modified content +diff --git a/Conditions/include/Conditions/SimpleTableCondition.h b/Conditions/include/Conditions/SimpleTableCondition.h +index 467d8b9..632de04 100644 +--- a/Conditions/include/Conditions/SimpleTableCondition.h ++++ b/Conditions/include/Conditions/SimpleTableCondition.h +@@ -8,6 +8,7 @@ + + #include + #include ++#include + + #include "Framework/ConditionsObject.h" + #include "Framework/Exception/Exception.h" diff --git a/ci/interop/v4.0.5.patch b/ci/interop/v4.0.5.patch new file mode 100644 index 0000000..62ff99c --- /dev/null +++ b/ci/interop/v4.0.5.patch @@ -0,0 +1,12 @@ +diff --git a/Conditions/include/Conditions/SimpleTableCondition.h b/Conditions/include/Conditions/SimpleTableCondition.h +index d34d930d..b69f81c7 100644 +--- a/Conditions/include/Conditions/SimpleTableCondition.h ++++ b/Conditions/include/Conditions/SimpleTableCondition.h +@@ -8,6 +8,7 @@ + + #include + #include ++#include + + #include "Framework/ConditionsObject.h" + #include "Framework/Exception/Exception.h" diff --git a/ci/interop/v4.1.4.patch b/ci/interop/v4.1.4.patch new file mode 100644 index 0000000..0f37d2e --- /dev/null +++ b/ci/interop/v4.1.4.patch @@ -0,0 +1,23 @@ +diff --git a/Conditions/include/Conditions/SimpleTableCondition.h b/Conditions/include/Conditions/SimpleTableCondition.h +index d34d930d..b69f81c7 100644 +--- a/Conditions/include/Conditions/SimpleTableCondition.h ++++ b/Conditions/include/Conditions/SimpleTableCondition.h +@@ -8,6 +8,7 @@ + + #include + #include ++#include + + #include "Framework/ConditionsObject.h" + #include "Framework/Exception/Exception.h" +diff --git a/TrigScint/src/TrigScint/Firmware/clusterproducer_sw.cxx b/TrigScint/src/TrigScint/Firmware/clusterproducer_sw.cxx +index 140011d9..0e07b681 100644 +--- a/TrigScint/src/TrigScint/Firmware/clusterproducer_sw.cxx ++++ b/TrigScint/src/TrigScint/Firmware/clusterproducer_sw.cxx +@@ -1,5 +1,6 @@ + #include + ++#include + #include + + #include "TrigScint/Firmware/clusterproducer.h" diff --git a/ci/interop/v4.2.3.patch b/ci/interop/v4.2.3.patch new file mode 120000 index 0000000..eec60da --- /dev/null +++ b/ci/interop/v4.2.3.patch @@ -0,0 +1 @@ +v4.1.4.patch \ No newline at end of file diff --git a/ci/ldmx-sw-dependabot b/ci/ldmx-sw-dependabot new file mode 100755 index 0000000..6506d2e --- /dev/null +++ b/ci/ldmx-sw-dependabot @@ -0,0 +1,98 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +error() { + printf >&2 "\033[1;31mERROR: \033[0m\033[31m%s\033[0m\n" "$*" +} + +inf() { + printf "\033[1mINFO: \033[0m%s\n" "$*" +} + +# fetch the latest release version for the input owner and repo +# https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-the-latest-release +# Args +# 1 : repository +# Output +# prints the tag to stdout +fetch() { + gh api \ + -H "Accept: application/vnd.github.json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "/repos/${1}/releases/latest" \ + --jq '.tag_name' +} + +# update the test workflow with the passed runner and version +# Args: +# 1 : provide to actually apply edit, otherwise just print lines that would change +# Output: +# Edits the ci/runners-to-test.json in place if 3 args provided, +# otherwise just prints lines that would be edited to stdout +update() { + latest="$(fetch LDMX-Software/ldmx-sw)" + ci_latest="$(jq -cr '.ldmx_sw[-1]' ci/ldmx-sw-to-test.json)" + if [ "${ci_latest}" = "${latest}" ]; then + inf "CI ${ci_latest} and ldmx-sw ${latest} agree on latest version. No updates to be made." + return 0 + fi + latest_major=$(echo "${latest}" | sed 's|^v||' | cut -f 1 -d .) + latest_minor=$(echo "${latest}" | cut -f 2 -d .) + latest_patch=$(echo "${latest}" | cut -f 3 -d .) + ci_latest_major=$(echo "${ci_latest}" | sed 's|^v||' | cut -f 1 -d .) + ci_latest_minor=$(echo "${ci_latest}" | cut -f 2 -d .) + ci_latest_patch=$(echo "${ci_latest}" | cut -f 3 -d .) + if [ "${latest_major}" -eq "${ci_latest_major}" ] && [ "${latest_minor}" -eq "${ci_latest_minor}" ]; then + inf "Only a patch difference between ${latest} and ${ci_latest}, replacing ${ci_latest} with ${latest}." + jq --arg latest "${latest}" '.ldmx_sw[-1] = $latest' ci/ldmx-sw-to-test.json > ci/ldmx-sw-to-test.json.upd + else + inf "More than a patch difference, adding ${latest} as a new test to the CI." + jq --arg latest "${latest}" '.ldmx_sw += [$latest]' ci/ldmx-sw-to-test.json > ci/ldmx-sw-to-test.json.upd + fi + if [ -z "${1}" ]; then + diff ci/ldmx-sw-to-test.json ci/ldmx-sw-to-test.json.upd || true + rm ci/ldmx-sw-to-test.json.upd + else + mv ci/ldmx-sw-to-test.json.upd ci/ldmx-sw-to-test.json + fi +} + +help() { + cat<<\HELP + + check for new runner versions to test + + USAGE: + + ./ci/ldmx-sw-dependabot [options] + + OPTIONS + -h, --help : print this help and exit + --apply : actually edit the ci/ldmx-sw-to-test.json instead of + just printing the lines that have been changed + +HELP +} + +apply="" +while [ "$#" -gt "0" ]; do + case "$1" in + -h|--help) + help + exit 0 + ;; + --apply) + apply="APPLY" + ;; + *) + help + error "Unrecognized argument '$1'" + exit 1 + ;; + esac + shift +done + +update "${apply}" diff --git a/ci/ldmx-sw-to-test.json b/ci/ldmx-sw-to-test.json new file mode 100644 index 0000000..4e40542 --- /dev/null +++ b/ci/ldmx-sw-to-test.json @@ -0,0 +1,10 @@ +{ + "ldmx_sw": [ + "trunk", + "v3.3.0", + "v3.4.0", + "v4.0.5", + "v4.1.4", + "v4.2.3" + ] +} diff --git a/ci/test-ldmx-sw b/ci/test-ldmx-sw new file mode 100755 index 0000000..54e2b7e --- /dev/null +++ b/ci/test-ldmx-sw @@ -0,0 +1,51 @@ +#!/bin/bash + +set -o errexit +set -o nounset + +echo "::group::initialize environment" +curl -s https://tomeichlersmith.github.io/denv/install | sh +denv check +denv init ${1} +echo "::endgroup::" + +echo "::group::clone ldmx-sw" +git clone --recursive --branch ${2} https://github.com/LDMX-Software/ldmx-sw.git +echo "::endgroup::" + +echo "::group::patch ldmx-sw" +patch_file="ci/interop/${2}.patch" +if [ -f "${patch_file}" ]; then + echo "applying ${patch_file}" + git -C ldmx-sw apply "../${patch_file}" + git -C ldmx-sw diff +else + echo "no patches found (${patch_file} does not exist)" +fi +echo "::endgroup::" + +ldmx_sw_major="$(echo "${2}" | sed 's|^v||' | cut -f 1 -d .)" +ldmx_sw_minor="$(echo "${2}" | cut -f 2 -d .)" + +echo "::group::build ldmx-sw" +use_just=false +if [ "${2}" = "trunk" ]; then + echo "we can use just since we are on trunk" + use_just=true +elif [ "${ldmx_sw_major}" -gt 4 ] || [ "${ldmx_sw_major}" -eq 4 ] && [ "${ldmx_sw_minor}" -gt 0 ]; then + echo "we can use just since we are on a release >= 4.1.0" + use_just=true +fi + +if ${use_just}; then + just ldmx-sw/configure + just ldmx-sw/build +else + denv cmake -B ldmx-sw/build -S ldmx-sw + denv cmake --build ldmx-sw/build --target install +fi +echo "::endgroup::" + +echo "::group::basic simulation run" +denv fire ldmx-sw/SimCore/test/basic.py +echo "::endgroup::"