Skip to content

Commit

Permalink
Merge branch '36676' into 36964
Browse files Browse the repository at this point in the history
  • Loading branch information
Matthias Koeppe committed Jun 9, 2024
2 parents df9dd71 + fd5ba9a commit 2bd68c9
Show file tree
Hide file tree
Showing 1,057 changed files with 6,693 additions and 4,847 deletions.
5 changes: 3 additions & 2 deletions .ci/retrofit-worktree.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@ set -x
# If actions/checkout downloaded our source tree using the GitHub REST API
# instead of with git (because do not have git installed in our image),
# we first make the source tree a repo.
if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
if [ ! -d .git ]; then git init; fi

# Tag this state of the source tree "new". This is what we want to build and test.
# Commit and tag this state of the source tree "new". This is what we want to build and test.
git add -A && git commit --quiet --allow-empty -m "new"
git tag -f new

# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
#!/usr/bin/env bash

echo 'set_labels_by_changes.sh called with environment:'
# cd to the PR directory
cd $PR_MOUNT_DIR
PR_BASE_SHA=$(git merge-base $BASE_SHA $PR_HEAD_SHA)

echo "set_labels_by_changes.sh called with environment:"
echo "BASE SHA: $PR_BASE_SHA"
echo "HEAD SHA: $PR_HEAD_SHA"
echo "SMALL THRESHOLD $SMALL_THRESHOLD"
Expand Down
5 changes: 3 additions & 2 deletions .ci/write-dockerfile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ cat <<EOF
FROM with-system-packages as bootstrapped
#:bootstrapping:
RUN rm -rf /new /sage/.git
$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 .gitignore /new/
$ADD config/config.rpath /new/config/config.rpath
$ADD src/doc/bootstrap /new/src/doc/bootstrap
Expand Down Expand Up @@ -338,7 +339,6 @@ ENV SAGE_CHECK=warn
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
$ADD .gitignore /new/.gitignore
$ADD src /new/src
ADD .ci /.ci
RUN cd /new && rm -rf .git && \
if /.ci/retrofit-worktree.sh worktree-pre /sage; then \
cd /sage && touch configure build/make/Makefile; \
Expand All @@ -347,7 +347,8 @@ RUN cd /new && rm -rf .git && \
rm -rf /sage/src; \
mv src /sage/src; \
cd /sage && ./bootstrap && ./config.status; \
fi
fi; \
cd /sage && rm -rf /new .git
ARG TARGETS="build"
$RUN $CHECK_STATUS_THEN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN $THEN_SAVE_STATUS
Expand Down
77 changes: 62 additions & 15 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ env:
jobs:
test-new:
runs-on: ubuntu-latest
outputs:
build_targets: ${{ steps.build-targets.outputs.build_targets }}
services:
# https://docs.docker.com/build/ci/github-actions/local-registry/
registry:
Expand All @@ -98,6 +100,46 @@ jobs:
- name: Checkout
id: checkout
uses: actions/checkout@v4
- name: Get changed files and packages
id: changed-files
uses: tj-actions/changed-files@v44
with:
# File extensions for doctests per sage.doctest.control.skipfile
files_yaml: |
configures:
- 'build/pkgs/*/spkg-configure.m4'
pkgs:
- 'build/pkgs/**'
- 'pkgs/**'
doctests:
- 'src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}'
- '!src/{setup,conftest*}.py'
- name: Determine targets to build
id: build-targets
run: |
uninstall_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))
build_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))
if [ -n "$uninstall_targets" ]; then
echo "build_targets=$uninstall_targets reconfigure $build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
else
echo "build_targets=$build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
fi
cat $GITHUB_OUTPUT
- uses: actions/checkout@v4
with:
ref: ${{ github.base_ref }}
path: worktree-base
if: github.base_ref && steps.changed-files.outputs.pkgs_all_changed_files
- name: Compute metrics
run: |
export PATH=build/bin:$PATH
if [ -d worktree-base ]; then
(echo "# $GITHUB_BASE_REF"; SAGE_ROOT=worktree-base sage-package metrics :all:) > base-metrics.txt
(echo "# $GITHUB_REF"; sage-package metrics :all:) > metrics.txt
diff --color=always --width=100 --side-by-side --left-column base-metrics.txt metrics.txt || true
else
sage-package metrics :all:
fi
- name: Install test prerequisites
# From docker.yml
run: |
Expand Down Expand Up @@ -142,7 +184,7 @@ jobs:
# more reliable than "load", for which we observed random failure
# conditions in which the built image could not be found.
#
push: true
push: ${{ steps.changed-files.outputs.doctests_all_changed_files && true || false }}
load: false
context: .
tags: ${{ env.BUILD_IMAGE }}
Expand All @@ -153,9 +195,12 @@ jobs:
NUMPROC=6
USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
TARGETS_PRE=build/make/Makefile
TARGETS=ci-build-with-fallback
TARGETS=${{ steps.build-targets.outputs.build_targets }}
- name: Start container
id: container
# Try to continue when "exporting to GitHub Actions Cache" failed with timeout
if: (success() || failure()) && steps.changed-files.outputs.doctests_all_changed_files
run: |
docker run --name BUILD -dit \
--mount type=bind,src=$(pwd),dst=$(pwd) \
Expand All @@ -165,6 +210,7 @@ jobs:
# Testing

- name: Check that all modules can be imported
if: (success() || failure()) && steps.container.outcome == 'success' && steps.changed-files.outputs.doctests_all_changed_files
run: |
# Increase the length of the lines in the "short summary"
export COLUMNS=120
Expand All @@ -175,20 +221,12 @@ jobs:
./sage -python -m pytest -c tox.ini -qq --doctest --collect-only || true
shell: sh .ci/docker-exec-script.sh BUILD /sage {0}

- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@v44
with:
# File extensions per sage.doctest.control.skipfile
files: src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}
files_ignore: src/{setup,conftest*}.py

- name: Test changed files (sage -t --new)
if: steps.changed-files.outputs.all_changed_files
if: (success() || failure()) && steps.container.outcome == 'success' && steps.changed-files.outputs.doctests_all_changed_files
run: |
export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4
# https://github.com/tj-actions/changed-files?tab=readme-ov-file#outputs-
./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.all_changed_files }}
./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.doctests_all_changed_files }}
shell: sh .ci/docker-exec-script.sh BUILD /sage {0}

test-mod:
Expand Down Expand Up @@ -266,9 +304,11 @@ jobs:
NUMPROC=6
USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
TARGETS_PRE=build/make/Makefile
TARGETS=ci-build-with-fallback
TARGETS=${{ needs.test-new.outputs.build_targets }}
- name: Start container
id: container
if: (success() || failure())
run: |
docker run --name BUILD -dit \
--mount type=bind,src=$(pwd),dst=$(pwd) \
Expand All @@ -278,6 +318,7 @@ jobs:
# Testing

- name: Test modularized distributions
if: (success() || failure()) && steps.container.outcome == 'success'
run: |
export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4
make V=0 tox-ensure && make ${{ matrix.targets }}
Expand Down Expand Up @@ -361,10 +402,11 @@ jobs:
NUMPROC=6
USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
TARGETS_PRE=build/make/Makefile
TARGETS=ci-build-with-fallback
TARGETS=${{ needs.test-new.outputs.build_targets }}
- name: Start container
id: container
if: (success() || failure())
run: |
docker run --name BUILD -dit \
--mount type=bind,src=$(pwd),dst=$(pwd) \
Expand All @@ -374,6 +416,7 @@ jobs:
# Testing

- name: Test all files (sage -t --long ${{ matrix.tests }})
if: (success() || failure()) && steps.container.outcome == 'success'
run: |
mkdir .coverage
rm -rf /sage/.coverage
Expand Down Expand Up @@ -473,10 +516,11 @@ jobs:
NUMPROC=6
USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
TARGETS_PRE=build/make/Makefile
TARGETS=ci-build-with-fallback
TARGETS=${{ needs.test-new.outputs.build_targets }}
- name: Start container
id: container
if: (success() || failure())
run: |
docker run --name BUILD -dit \
--mount type=bind,src=$(pwd),dst=$(pwd) \
Expand All @@ -490,8 +534,10 @@ jobs:
with:
path: .coverage
pattern: coverage-*
if: (success() || failure()) && steps.container.outcome == 'success'

- name: Coverage report
if: (success() || failure()) && steps.container.outcome == 'success'
# Using --omit to avoid "CoverageWarning: Couldn't parse '/tmp/tmp06qizzie/tmp_ldpu46ob.py': No source for code"
run: |
rm -rf /sage/.coverage
Expand All @@ -505,6 +551,7 @@ jobs:
shell: sh .ci/docker-exec-script.sh BUILD . {0}

- name: Upload coverage to codecov
if: (success() || failure()) && steps.container.outcome == 'success'
uses: codecov/codecov-action@v4
with:
directory: .coverage/coverage-report
31 changes: 13 additions & 18 deletions .github/workflows/ci-linux-incremental.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,11 @@ jobs:
runs-on: ubuntu-latest
name: List changed packages
outputs:
uninstall_targets: ${{ steps.build-targets.outputs.uninstall_targets }}
build_targets: ${{ steps.build-targets.outputs.build_targets }}
steps:
- uses: actions/checkout@v4
- name: Get all packages that have changed
id: changed-packages
id: changed-files
uses: tj-actions/changed-files@v44
with:
files_yaml: |
Expand All @@ -58,14 +57,19 @@ jobs:
- name: Determine targets to build
id: build-targets
run: |
echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))" >> $GITHUB_OUTPUT
uninstall_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))
build_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))
if [ -n "$uninstall_targets" ]; then
echo "build_targets=$uninstall_targets reconfigure $build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
else
echo "build_targets=$build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
fi
cat $GITHUB_OUTPUT
- uses: actions/checkout@v4
with:
ref: ${{ github.base_ref }}
path: worktree-base
if: github.base_ref
if: github.base_ref && steps.changed-files.outputs.pkgs_all_changed_files
- name: Compute metrics
run: |
export PATH=build/bin:$PATH
Expand All @@ -88,15 +92,14 @@ jobs:
from_docker_target: "with-targets"
from_docker_tag: "dev"
docker_targets: "with-targets"
targets: "${{needs.changed_files.outputs.uninstall_targets}} reconfigure ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
targets: "${{needs.changed_files.outputs.build_targets}} doc-html ptest-nodoc"
tox_system_factors: >-
["ubuntu-focal",
"ubuntu-jammy",
"ubuntu-mantic",
"ubuntu-noble",
"debian-bullseye",
"debian-bookworm",
"fedora-30",
"fedora-38",
"fedora-40",
"gentoo-python3.11",
"debian-bullseye-i386"]
tox_packages_factors: >-
Expand All @@ -107,14 +110,6 @@ jobs:

site:
needs: [changed_files]
if: |
github.event_name != 'pull_request' ||
((github.event.action != 'labeled' &&
(contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
contains(github.event.pull_request.labels.*.name, 'c: packages: optional'))) ||
(github.event.action == 'labeled' &&
(github.event.label.name == 'c: packages: optional' ||
github.event.label.name == 'c: packages: standard')))
uses: ./.github/workflows/docker.yml
with:
# Build incrementally from published Docker image
Expand All @@ -124,7 +119,7 @@ jobs:
from_docker_target: "with-targets"
from_docker_tag: "dev"
docker_targets: "with-targets"
targets: "${{needs.changed_files.outputs.uninstall_targets}} reconfigure ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
targets: "${{needs.changed_files.outputs.build_targets}} doc-html ptest-nodoc"
# Only test systems with a usable system python (>= 3.9)
# with only a small number of test failures as of 10.2.rc0
tox_system_factors: >-
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/doc-build-pdf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ jobs:
TARGETS=ci-build-with-fallback
- name: Start container
id: container
# Try to continue when "exporting to GitHub Actions Cache" failed with timeout
if: (success() || failure())
run: |
docker run --name BUILD -dit \
--mount type=bind,src=$(pwd),dst=$(pwd) \
Expand All @@ -110,6 +113,8 @@ jobs:
# Docs

- name: Update system packages
id: packages
if: (success() || failure()) && steps.container.outcome == 'success'
run: |
export PATH="build/bin:$PATH"
eval $(sage-print-system-package-command auto update)
Expand All @@ -119,6 +124,7 @@ jobs:

- name: Build docs (PDF)
id: docbuild
if: (success() || failure()) && steps.packages.outcome == 'success'
run: |
export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/doc-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ jobs:
TARGETS=ci-build-with-fallback
- name: Start container
id: container
# Try to continue when "exporting to GitHub Actions Cache" failed with timeout
if: (success() || failure())
run: |
docker run --name BUILD -dit \
--mount type=bind,src=$(pwd),dst=$(pwd) \
Expand All @@ -115,6 +118,7 @@ jobs:

- name: Store old docs
id: worktree
if: (success() || failure()) && steps.container.outcome == 'success'
run: |
git config --global --add safe.directory $(pwd)
git config --global user.email "ci-sage@example.com"
Expand All @@ -129,6 +133,7 @@ jobs:
# Wipe out chronic diffs between old doc and new doc
(cd docs && \
find . -name "*.html" | xargs sed -i -e '/class="sidebar-brand-text"/ s/Sage [0-9a-z.]* /Sage '"$new_version"' /' \
-e '/This is documentation for/ s/ built with GitHub PR [^.]*././' \
-e 's;'"$mathjax_path_from"';'"$mathjax_path_to"';' \
-e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d')
# Create git repo from old doc
Expand All @@ -140,9 +145,12 @@ jobs:
- name: Build docs
id: docbuild
if: (success() || failure()) && steps.worktree.outcome == 'success'
# Always non-incremental because of the concern that
# incremental docbuild may introduce broken links (inter-file references) though build succeeds
run: |
export GITHUB_REF=${{ github.ref }}
export PR_SHA=${{ github.event.pull_request.head.sha }}
export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
make doc-clean doc-uninstall
export SAGE_USE_CDNS=yes
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/doc-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
NETLIFY_PRODUCTION: ${{ github.ref == 'refs/heads/develop' }}
NETLIFY_MESSAGE: ${{ steps.source-run-info.outputs.pullRequestNumber }}
NETLIFY_ALIAS: deploy-preview-${{ steps.source-run-info.outputs.pullRequestNumber }}
NETLIFY_ALIAS: preview-${{ steps.source-run-info.outputs.pullRequestNumber }}

# Add deployment as status check, PR comment and annotation
# we could use the nwtgck/actions-netlify action for that, except for that it is not (yet) working in workflow_run context: https://github.com/nwtgck/actions-netlify/issues/545
Expand Down Expand Up @@ -130,7 +130,7 @@ jobs:
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
NETLIFY_MESSAGE: Deployed live doc
NETLIFY_ALIAS: deploy-livedoc
NETLIFY_ALIAS: livedoc

- name: Report deployment url
run: |
Expand Down
Loading

0 comments on commit 2bd68c9

Please sign in to comment.