diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 89ed87ecf91..5242dfc06f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,20 +85,19 @@ jobs: (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch fi - - name: Incremental build, test changed files (sage -t --new) + - name: Incremental build id: incremental run: | # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. - # We run tests with "sage -t --new"; this only tests the uncommitted changes. - ./bootstrap && make build && ./sage -t --new -p2 + ./bootstrap && make build working-directory: ./worktree-image env: MAKE: make -j2 --output-sync=recurse SAGE_NUM_THREADS: 2 - - name: Build and test modularized distributions + - name: Build modularized distributions if: always() && steps.worktree.outcome == 'success' - run: make V=0 tox && make pypi-wheels + run: make V=0 tox && make SAGE_CHECK=no pypi-wheels working-directory: ./worktree-image env: MAKE: make -j2 --output-sync=recurse @@ -143,6 +142,26 @@ jobs: MAKE: make -j2 --output-sync=recurse SAGE_NUM_THREADS: 2 + # Testing + + - name: Test changed files (sage -t --new) + if: always() && steps.build.outcome == 'success' + run: | + # We run tests with "sage -t --new"; this only tests the uncommitted changes. + ./sage -t --new -p2 + working-directory: ./worktree-image + env: + MAKE: make -j2 --output-sync=recurse + SAGE_NUM_THREADS: 2 + + - name: Test modularized distributions + if: always() && steps.build.outcome == 'success' + run: make V=0 tox && make pypi-wheels-check + working-directory: ./worktree-image + env: + MAKE: make -j2 --output-sync=recurse + SAGE_NUM_THREADS: 2 + - name: Pytest if: contains(github.ref, 'pytest') run: | diff --git a/build/bin/sage-dist-helpers b/build/bin/sage-dist-helpers index 67a2201d31f..73119347438 100644 --- a/build/bin/sage-dist-helpers +++ b/build/bin/sage-dist-helpers @@ -321,7 +321,15 @@ sdh_store_wheel() { mkdir -p "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}" && \ $sudo mv "$wheel" "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/" || \ sdh_die "Error storing $wheel" - wheel="${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/${wheel##*/}" + wheel="${SAGE_SPKG_WHEELS}/${wheel##*/}" + if [ -n "${SAGE_SPKG_SCRIPTS}" -a -n "${PKG_BASE}" ]; then + wheel_basename="${wheel##*/}" + distname="${wheel_basename%%-*}" + # Record name and wheel file location + mkdir -p ${SAGE_DESTDIR}${SAGE_SPKG_SCRIPTS}/${PKG_BASE} + echo "${distname} @ file://${wheel}" >> ${SAGE_DESTDIR}${SAGE_SPKG_SCRIPTS}/${PKG_BASE}/spkg-requirements.txt + fi + wheel="${SAGE_DESTDIR}${wheel}" } sdh_store_and_pip_install_wheel() { @@ -373,11 +381,6 @@ sdh_store_and_pip_install_wheel() { fi $sudo sage-pip-install $root $pip_options "$wheel" || \ sdh_die "Error installing ${wheel##*/}" - if [ -n "${SAGE_PKG_DIR}" ]; then - # Record name of installed distribution name for uninstallation. - wheel=${wheel##*/} - echo "${wheel%%-*}" >> ${SAGE_PKG_DIR}/spkg-piprm-requirements.txt - fi } sdh_pip_uninstall() { diff --git a/build/bin/sage-spkg b/build/bin/sage-spkg index 9b38ed6afc7..b540c8e866f 100755 --- a/build/bin/sage-spkg +++ b/build/bin/sage-spkg @@ -548,7 +548,7 @@ WRAPPED_SCRIPTS="build install check preinst postinst $INSTALLED_SCRIPTS" # Prepare script for uninstallation of packages that use sdh_pip_install # or sdh_store_and_pip_install_wheel. -echo 'sdh_pip_uninstall -r $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-piprm-requirements.txt' > spkg-piprm.in +echo 'sdh_pip_uninstall -r $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-requirements.txt' > spkg-piprm.in for script in $WRAPPED_SCRIPTS; do # 'Installed' scripts are not run immediately out of the package build @@ -567,24 +567,6 @@ for script in $WRAPPED_SCRIPTS; do fi done - -# When there is no spkg-install, assume the "spkg" is a tarball not -# specifically made for Sage. Since we want it to be as easy as -# possible to install such a package, we "guess" spkg-install. -if [ ! -f spkg-install ]; then - echo '#!/usr/bin/env bash' > spkg-install - if [ -x configure ]; then - echo './configure --prefix="$SAGE_INST_LOCAL" && make && $SAGE_SUDO make install' >> spkg-install - elif [ -f setup.py ]; then - echo 'python setup.py install' >> spkg-install - else - echo >&2 "Error: There is no spkg-install script, no setup.py, and no configure" - echo >&2 "script, so I do not know how to install $PKG_SRC." - exit 1 - fi - chmod +x spkg-install -fi - echo "****************************************************" echo "Host system:" uname -a @@ -722,9 +704,7 @@ unset SAGE_DESTDIR_LOCAL # removed by sage-spkg-uninstall INSTALLED_SCRIPTS_DEST="$SAGE_SPKG_SCRIPTS/$PKG_BASE" -if [ -f spkg-piprm-requirements.txt ]; then - INSTALLED_SCRIPTS="$INSTALLED_SCRIPTS piprm-requirements.txt" -else +if [ ! -f $INSTALLED_SCRIPTS_DEST/spkg-requirements.txt ]; then # No packages to uninstall with pip, so remove the prepared uninstall script rm -f spkg-piprm spkg-piprm.in fi diff --git a/build/make/Makefile.in b/build/make/Makefile.in index cc004d08c3c..6031ae7fa2a 100644 --- a/build/make/Makefile.in +++ b/build/make/Makefile.in @@ -468,6 +468,8 @@ wheels: $(MAKE_REC) SAGE_EDITABLE=no SAGE_WHEELS=yes $(WHEEL_PACKAGES) @echo "Built wheels are in venv/var/lib/sage/wheels/" +pypi-wheels-check: $(PYPI_WHEEL_PACKAGES:%=%-check) + #============================================================================== # Setting SAGE_CHECK... variables #============================================================================== @@ -724,15 +726,21 @@ $(1)-$(4)-no-deps: echo "$$($(4)_DISABLED_MESSAGE)" 2>&1; \ exit 1; \ elif [ -x '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install' ]; then \ + rm -rf '$$($(4))/var/lib/sage/scripts/$(1)'; \ cd '$$(SAGE_ROOT)/build/pkgs/$(1)' && \ . '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \ . '$$(SAGE_ROOT)/src/bin/sage-env-config' && \ . '$$(SAGE_ROOT)/src/bin/sage-env' && \ . '$$(SAGE_ROOT)/build/bin/sage-build-env-config' && \ . '$$(SAGE_ROOT)/build/bin/sage-build-env' && \ + PKG_BASE="$(1)" \ + PKG_VER="$(2)" \ + PKG_NAME="$(1)-$(2)" \ SAGE_SPKG_WHEELS=$$($(4))/var/lib/sage/wheels \ + SAGE_SPKG_SCRIPTS=$$($(4))/var/lib/sage/scripts \ SAGE_INST_LOCAL=$$($(4)) \ - sage-logger -p 'SAGE_CHECK=$$(SAGE_CHECK_$(1)) $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install' '$$(SAGE_LOGS)/$(1)-$(2).log' && \ + SAGE_CHECK=$$(SAGE_CHECK_$(1)) \ + sage-logger -p '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install && if [ $$$$SAGE_CHECK != no -a -x $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check ]; then $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check; fi' '$$(SAGE_LOGS)/$(1)-$(2).log' && \ rm -f "$$($(4))/$(SPKG_INST_RELDIR)/$(1)"-* && \ touch "$$($(4))/$(SPKG_INST_RELDIR)/$(1)-$(2)"; \ else ( \ @@ -752,6 +760,26 @@ $(1)-$(4)-no-deps: $(1)-no-deps: $(1)-$(4)-no-deps +$(1)-$(4)-check: + $(PLUS)@if [ -x $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check ]; then \ + cd '$$(SAGE_ROOT)/build/pkgs/$(1)' && \ + . '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \ + . '$$(SAGE_ROOT)/src/bin/sage-env-config' && \ + . '$$(SAGE_ROOT)/src/bin/sage-env' && \ + . '$$(SAGE_ROOT)/build/bin/sage-build-env-config' && \ + . '$$(SAGE_ROOT)/build/bin/sage-build-env' && \ + PKG_BASE="$(1)" \ + PKG_VER="$(2)" \ + PKG_NAME="$(1)-$(2)" \ + SAGE_SPKG_WHEELS=$$($(4))/var/lib/sage/wheels \ + SAGE_SPKG_SCRIPTS=$$($(4))/var/lib/sage/scripts \ + SAGE_INST_LOCAL=$$($(4)) \ + SAGE_CHECK=$$(SAGE_CHECK_$(1)) \ + sage-logger -p '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check' '$$(SAGE_LOGS)/$(1)-$(2).log'; \ + fi + +$(1)-check: $(1)-$(4)-check + $(1)-$(4)-uninstall: -$(AM_V_at)cd '$$(SAGE_ROOT)/build/pkgs/$(1)' && \ . '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \ diff --git a/build/pkgs/sage_sws2rst/spkg-check b/build/pkgs/sage_sws2rst/spkg-check index 667df773214..7372750871e 100755 --- a/build/pkgs/sage_sws2rst/spkg-check +++ b/build/pkgs/sage_sws2rst/spkg-check @@ -1,2 +1,11 @@ #! /bin/sh -cd src && ./check.sh +cd src +./check.sh +if [ $? -ne 0 ]; then + if [ "$SAGE_CHECK" = "warn" ]; then + echo >&2 "Warning: Failures testing package $PKG_NAME (ignored)" + else + echo >&2 "Error testing package $PKG_NAME" + exit 1 + fi +fi diff --git a/build/pkgs/sage_sws2rst/spkg-install b/build/pkgs/sage_sws2rst/spkg-install index 9cb71471fd8..a180fb36542 100755 --- a/build/pkgs/sage_sws2rst/spkg-install +++ b/build/pkgs/sage_sws2rst/spkg-install @@ -18,13 +18,3 @@ if [ "$SAGE_EDITABLE" = yes ]; then else sdh_pip_install . fi -cd .. -# For type=script packages, spkg-check is not run -case "$SAGE_CHECK" in - yes) - ./spkg-check - ;; - warn) - ./spkg-check || echo >&2 "Warning: Failures testing package sage_sws2rst (ignored)" - ;; -esac diff --git a/build/pkgs/sagemath_categories/spkg-check b/build/pkgs/sagemath_categories/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_categories/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_environment/spkg-check b/build/pkgs/sagemath_environment/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_environment/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file diff --git a/build/pkgs/sagemath_objects/spkg-check b/build/pkgs/sagemath_objects/spkg-check new file mode 100755 index 00000000000..87cea0fc38f --- /dev/null +++ b/build/pkgs/sagemath_objects/spkg-check @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +cd src + +export PIP_NO_INDEX=true +export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS" + +export TOX_PARALLEL_NO_SPINNER=1 +wheel="$(sed -n '1s,.*@ file://,,p' $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-requirements.txt)" +echo Running "tox -r -p auto -v --installpkg $wheel" +tox -r -p auto -v --installpkg "$wheel" +status=$? +case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in + 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";; + 0:*:*) echo "Passed the test suite";; + *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;; + *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;; + *:yes:0) echo "New failures, not in baseline known-test-failures*.json";; + *:yes:*) echo "Failures testing the package";; +esac +# Show summaries of failures (suppress lines ending with '[failed in baseline]') +for f in $(pwd)/.tox/sagepython-sagewheels-nopypi-norequirements*/log/*-command*.log; do + if [ -r "$f" ]; then + echo "$f" + grep '^sage -t.*#[^]]*$' "$f" + fi +done +exit $status diff --git a/build/pkgs/sagemath_objects/spkg-install b/build/pkgs/sagemath_objects/spkg-install index 472e7f0d4d6..18ceb18724d 100755 --- a/build/pkgs/sagemath_objects/spkg-install +++ b/build/pkgs/sagemath_objects/spkg-install @@ -23,26 +23,3 @@ python3 -m build --outdir "$DIST_DIR"/dist . || sdh_die "Failure building sdist wheel=$(cd "$DIST_DIR" && sdh_store_wheel . >&2 && echo $wheel) ls -l "$wheel" - -if [ "$SAGE_CHECK" != no ]; then - export TOX_PARALLEL_NO_SPINNER=1 - echo Running "tox -r -p auto -v --installpkg $wheel" - tox -r -p auto -v --installpkg $wheel - status=$? - case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in - 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";; - 0:*:*) echo "Passed the test suite";; - *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;; - *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;; - *:yes:0) echo "New failures, not in baseline known-test-failures*.json";; - *:yes:*) echo "Failures testing the package";; - esac - # Show summaries of failures (suppress lines ending with '[failed in baseline]') - for f in $(pwd)/.tox/sagepython-sagewheels-nopypi-norequirements*/log/*-command*.log; do - if [ -r "$f" ]; then - echo "$f" - grep '^sage -t.*#[^]]*$' "$f" - fi - done - exit $status -fi diff --git a/build/pkgs/sagemath_repl/spkg-check b/build/pkgs/sagemath_repl/spkg-check new file mode 120000 index 00000000000..91c6b1835fc --- /dev/null +++ b/build/pkgs/sagemath_repl/spkg-check @@ -0,0 +1 @@ +../sagemath_objects/spkg-check \ No newline at end of file