From a9442fbb9e535dfe76c4aee5e874e85a087b8ec6 Mon Sep 17 00:00:00 2001 From: Thomas Sibley Date: Thu, 11 May 2023 10:26:23 -0700 Subject: [PATCH] ci: Test pathogen repo CI builds with the final packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ Commit message based on that of 12000a20 in nextstrain/docker-base.¹ Code changes also based on that commit, plus subsequent commits.² ] A useful check for if new packages will break our pathogen builds. I included all pathogen repos that already use our pathogen-repo-ci reusable workflow. It should be minimal effort to maintain this list over time—I expect it to only grow—but perhaps in the future we will want to abstract it out into a shared list of known pathogen repos. I don't like that we have to copy the build-args for a few of the repos here since it'll be easy for this copy to diverge from the repo's authoritative build-args, but it's necessary for now. Over time as we work towards increased automation of pathogen builds, I think we can get rid of this build-args copy by further standardizing how each repo configures itself for automation. For example, instead of specifying build-args in a repo's CI workflow, the args for CI could be stored in a broader workflow metadata file (e.g. nextstrain-workflow.yaml) read by pathogen-repo-ci, or defined by some other convention. An alternative to directly running pathogen-repo-ci against each repo here would be instead triggering the CI workflows themselves within each repo. The downside to that is it would divorce the outcomes of those workflows from this one and render them not visible from PRs in this repo. It would also require updates to each repo to support triggering and passing in of additional parameters (i.e. for the package). And finally those CI workflows sometimes run other jobs, like linting and other integration tests (e.g. with Cram), that aren't always necessary to run with a new package. Related-to: Related-to: Related-to: ¹ ² --- .github/workflows/ci.yaml | 46 ++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7804b9b..e58e89b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,10 +18,13 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - - id: set-version - run: echo "version=$(./devel/generate-version)" | tee -a "$GITHUB_OUTPUT" + - id: set + run: | + echo "version=$(./devel/generate-version)" | tee -a "$GITHUB_OUTPUT" + echo "label=$(./devel/label-for-ref "$GITHUB_REF")" | tee -a "$GITHUB_OUTPUT" outputs: - version: ${{ steps.set-version.outputs.version }} + version: ${{ steps.set.outputs.version }} + label: ${{ steps.set.outputs.label }} build-and-test: needs: generate-version @@ -78,8 +81,6 @@ jobs: ./devel/diff-pkgs nextstrain-base-*.conda build/locked/*/nextstrain-base-*.conda \ > "$GITHUB_STEP_SUMMARY" - # XXX TODO: Install Nextstrain CLI (standalone), then run test build. - release: needs: - generate-version @@ -98,12 +99,10 @@ jobs: - run: ./devel/setup - - name: Set LABEL - run: echo LABEL="$(./devel/label-for-ref "$GITHUB_REF")" | tee -a "$GITHUB_ENV" - - run: ./devel/upload env: ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + LABEL: ${{ needs.generate-version.outputs.label }} - name: Generate summary run: | @@ -113,3 +112,34 @@ jobs: > "$GITHUB_STEP_SUMMARY" env: VERSION: ${{ needs.generate-version.outputs.version }} + LABEL: ${{ needs.generate-version.outputs.label }} + + # Run pathogen repo CI builds with the final packages + test-pathogen-repo-ci: + needs: + - generate-version + - release + strategy: + # XXX TODO: Test on multiple platforms (os, maybe arch) via the matrix too + matrix: + include: + - { pathogen: avian-flu, build-args: auspice/flu_avian_h5n1_ha.json } + - { pathogen: ebola } + - { pathogen: lassa } + - { pathogen: monkeypox } + - { pathogen: mumps } + - { pathogen: ncov, build-args: all_regions -j 2 --profile nextstrain_profiles/nextstrain-ci } + - { pathogen: seasonal-flu, build-args: --configfile profiles/ci/builds.yaml -p } + - { pathogen: zika } + name: test-pathogen-repo-ci (${{ matrix.pathogen }}) + uses: nextstrain/.github/.github/workflows/pathogen-repo-ci.yaml@master + with: + repo: nextstrain/${{ matrix.pathogen }} + build-args: ${{ matrix.build-args }} + runtimes: | + - conda + env: | + NEXTSTRAIN_CONDA_CHANNEL: nextstrain/label/${{ needs.generate-version.outputs.label }} + NEXTSTRAIN_CONDA_BASE_PACKAGE: nextstrain-base ==${{ needs.generate-version.outputs.version }} + artifact-name: ${{ matrix.pathogen }}-outputs + continue-on-error: true