Trac #33817: GH Actions: Add test of the modularized distributions

... as part of the workflow `build.yml`. This is to catch modularization
regressions on tickets.

To this end, we extend
`build/pkgs/sagemath-{objects,categories,environment,repl}/spkg-install`
scripts so that they build and store wheels in `SAGE_SPKG_WHEELS` (but
do not install them!), and then run `tox` on the wheels (unless disabled
by setting `SAGE_CHECK=no`).

A new top-level make target `pypi-wheels` runs these scripts for these
and other small distribution packages, ensuring that wheels are present
even if `configure --enable-editable` is in use.

Another top-level make target `wheels` does the same but also includes
`sage_conf` and `sagemath_standard`. This is useful for venv installs
and helps with the problem noted in #32913.

We also extend the script `src/bin/sage-update-version` so that it
automatically updates the `install-requires.txt` files for our
distribution packages.

URL: https://trac.sagemath.org/33817
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Tobias Diez, Kwankyu Lee

.github/workflows/build.yml

@@ -25,18 +25,11 @@ jobs:
  container: ghcr.io/sagemath/sage/sage-docker-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
  steps:
  - name: Checkout
+ id: checkout
  uses: actions/checkout@v2
 
- - name: Set up node to install pyright
- uses: actions/setup-node@v1
- with:
- node-version: '12'
-
- - name: Install pyright
- # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239
- run: npm install -g pyright@1.1.232
-
  - name: Prepare
+ id: prepare
  run: |
  # Install test tools.
  if apt-get update && apt-get install -y git python3-venv; then
@@ -53,10 +46,31 @@ jobs:
  ./bootstrap
  ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-editable --enable-download-from-upstream-url
  
+ - name: Build and test modularized distributions
+ if: always() && steps.prepare.outcome == 'success'
+ run: make V=0 tox && make pypi-wheels
+ env:
+ MAKE: make -j2
+ SAGE_NUM_THREADS: 2
+
+ - name: Set up node to install pyright
+ if: always() && steps.prepare.outcome == 'success'
+ uses: actions/setup-node@v1
+ with:
+ node-version: '12'
+
+ - name: Install pyright
+ if: always() && steps.prepare.outcome == 'success'
+ # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239
+ run: npm install -g pyright@1.1.232
+
  - name: Static code check with pyright
+ if: always() && steps.prepare.outcome == 'success'
  run: pyright 
 
  - name: Build
+ id: build
+ if: always() && steps.prepare.outcome == 'success'
  run: make build
  env:
  MAKE: make -j2
@@ -73,20 +87,21 @@ jobs:
  COLUMNS: 120
 
  - name: Test
+ if: always() && steps.build.outcome == 'success'
  run: |
  ../sage -python -m pip install coverage
  ../sage -python -m coverage run ./bin/sage-runtests --all -p2
  working-directory: ./src
 
  - name: Prepare coverage results
- if: always()
+ if: always() && steps.build.outcome == 'success'
  run: |
  ./venv/bin/python3 -m coverage combine src/.coverage/
  ./venv/bin/python3 -m coverage xml
  find . -name *coverage*
  
  - name: Upload coverage to codecov
- if: always()
+ if: always() && steps.build.outcome == 'success'
  uses: codecov/codecov-action@v2
  with:
  files: ./coverage.xml

Makefile

@@ -92,6 +92,30 @@ pypi-sdists: sage_setup
  ./sage --sh build/pkgs/sagemath_repl/spkg-src
  @echo "Built sdists are in upstream/"
 
+# Ensuring wheels are present, even for packages that may have been installed
+# as editable. Until we have better uninstallation of script packages, we
+# just remove the timestamps, which will lead to rebuilds of the packages.
+PYPI_WHEEL_PACKAGES = sage_sws2rst sage_setup sagemath_environment sagemath_objects sagemath_repl sagemath_categories
+pypi-wheels:
+ for a in $(PYPI_WHEEL_PACKAGES); do \
+ rm -f venv/var/lib/sage/installed/$$a-*; \
+ done
+ for a in $(PYPI_WHEEL_PACKAGES); do \
+ $(MAKE) SAGE_EDITABLE=no $$a; \
+ done
+ @echo "Built wheels are in venv/var/lib/sage/wheels/"
+
+# sage_docbuild is here, not in PYPI_WHEEL_PACKAGES, because it depends on sagelib
+WHEEL_PACKAGES = $(PYPI_WHEEL_PACKAGES) sage_conf sagelib sage_docbuild
+wheels:
+ for a in $(WHEEL_PACKAGES); do \
+ rm -f venv/var/lib/sage/installed/$$a-*; \
+ done
+ for a in $(WHEEL_PACKAGES); do \
+ $(MAKE) SAGE_EDITABLE=no $$a; \
+ done
+ @echo "Built wheels are in venv/var/lib/sage/wheels/"
+
 ###############################################################################
 # Cleaning up
 ###############################################################################
@@ -356,6 +380,7 @@ list:
  @$(MAKE) --silent -f build/make/Makefile SAGE_PKGCONFIG=dummy $@
 
 .PHONY: default build dist install micro_release \
+ pypi-sdists pypi-wheels wheels \
  misc-clean bdist-clean distclean bootstrap-clean maintainer-clean \
  test check testoptional testall testlong testoptionallong testallong \
  ptest ptestoptional ptestall ptestlong ptestoptionallong ptestallong \

build/bin/sage-build-env

@@ -27,6 +27,10 @@ if [ "x$SAGE_BUILD_ENV_SOURCED" = "x" ]; then
  if [ "x$SAGE_DEBUG" = "x" ]; then
  export SAGE_DEBUG="$CONFIGURED_SAGE_DEBUG"
  fi
+ # Likewise for SAGE_EDITABLE
+ if [ "x$SAGE_EDITABLE" = "x" ]; then
+ export SAGE_EDITABLE="$CONFIGURED_SAGE_EDITABLE"
+ fi
 
  # This is usually blank if the system GMP is used, or $SAGE_LOCAL otherwise
  if [ -n "$SAGE_GMP_PREFIX" ]; then

build/bin/sage-build-env-config.in

@@ -57,4 +57,4 @@ export SAGE_SUITESPARSE_PREFIX="@SAGE_SUITESPARSE_PREFIX@"
 
 export SAGE_CONFIGURE_FFLAS_FFPACK="@SAGE_CONFIGURE_FFLAS_FFPACK@"
 
-export SAGE_EDITABLE="@SAGE_EDITABLE@"
+export CONFIGURED_SAGE_EDITABLE="@SAGE_EDITABLE@"

build/pkgs/python_build/SPKG.rst

@@ -0,0 +1,18 @@
+python_build: A simple, correct PEP517 package builder
+======================================================
+
+Description
+-----------
+
+``build`` is a simple, correct PEP517 package builder
+
+License
+-------
+
+MIT
+
+Upstream Contact
+----------------
+
+https://pypi.org/project/build/
+

build/pkgs/python_build/dependencies

@@ -0,0 +1,4 @@
+$(PYTHON) pyparsing tomli packaging | $(PYTHON_TOOLCHAIN)
+
+----------
+All lines of this file are ignored except the first.

build/pkgs/python_build/requirements.txt

@@ -0,0 +1 @@
+build

build/pkgs/python_build/type

@@ -0,0 +1 @@
+optional

build/pkgs/sage_conf/install-requires.txt

@@ -1 +1,2 @@
-sage-conf ~= 9.7.b3
+# This file is updated on every release by the sage-update-version script
+sage-conf ~= 9.7b6

build/pkgs/sage_docbuild/install-requires.txt

@@ -1 +1,2 @@
-sage_docbuild
+# This file is updated on every release by the sage-update-version script
+sage-docbuild ~= 9.7b6

build/pkgs/sage_setup/install-requires.txt

@@ -1 +1,2 @@
-sage-setup ~= 9.7.b3
+# This file is updated on every release by the sage-update-version script
+sage-setup ~= 9.7b6

build/pkgs/sage_sws2rst/install-requires.txt

@@ -1 +1,2 @@
-sage_sws2rst
+# This file is updated on every release by the sage-update-version script
+sage-sws2rst ~= 9.7b6

build/pkgs/sagelib/install-requires.txt

@@ -1 +1,2 @@
-sagemath-standard
+# This file is updated on every release by the sage-update-version script
+sagelib ~= 9.7b6

build/pkgs/sagemath_categories/install-requires.txt

@@ -1 +1,2 @@
-sagemath-categories ~= 9.5b6
+# This file is updated on every release by the sage-update-version script
+sagemath-categories ~= 9.7b6

build/pkgs/sagemath_environment/install-requires.txt

@@ -1 +1,2 @@
-sagemath-environment == 9.6rc3.post2
+# This file is updated on every release by the sage-update-version script
+sagemath-environment ~= 9.7b6

build/pkgs/sagemath_objects/dependencies

@@ -1,4 +1,4 @@
-FORCE $(PYTHON) cysignals gmpy2 ipython | $(PYTHON_TOOLCHAIN) cython pkgconfig
+FORCE $(PYTHON) cysignals gmpy2 ipython | $(PYTHON_TOOLCHAIN) sage_setup cython pkgconfig python_build
 
 # FORCE: Always run the spkg-install script
 # ipython - for the doctester

build/pkgs/sagemath_objects/install-requires.txt

@@ -1,2 +1,2 @@
-# Pinned in Trac #29941 until proper namespace packages are supported in #28925.
-sagemath-objects == 9.6rc3.post4
+# This file is updated on every release by the sage-update-version script
+sagemath-objects ~= 9.7b6

build/pkgs/sagemath_objects/spkg-install

@@ -1,8 +1,33 @@
 #!/usr/bin/env bash
+# From sage-spkg.
+# For type=script packages, the build rule in build/make/Makefile sources
+# sage-env but not sage-dist-helpers.
+lib="$SAGE_ROOT/build/bin/sage-dist-helpers"
+source "$lib"
+if [ $? -ne 0 ]; then
+ echo >&2 "Error: failed to source $lib"
+ echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?"
+ exit 1
+fi
 cd src
 
+export PIP_NO_INDEX=true
+export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS"
+
+# First build the sdist, then build the wheel from the sdist.
+# https://pypa-build.readthedocs.io/en/latest/#python--m-build
+# (Important because sagemath-objects uses MANIFEST.in for filtering.)
+# Do not install the wheel.
+DIST_DIR="$(mktemp -d)"
+if ! python3 -m build --outdir "$DIST_DIR"/dist .; then
+ # This happens on Debian without python3-venv installed - "ensurepip" is missing
+ echo "Falling back to --no-isolation"
+ python3 -m build --no-isolation --outdir "$DIST_DIR"/dist . || sdh_die "Failure building sdist and wheel"
+fi
+
+wheel=$(cd "$DIST_DIR" && sdh_store_wheel . && echo $wheel)
+ls -l "$wheel"
+
 if [ "$SAGE_CHECK" != no ]; then
- tox
+ tox -v -e sagepython-norequirements --installpkg "$wheel"
 fi
-# We skip the install for now.
-exit 0

build/pkgs/tox/install-requires.txt

@@ -1 +1 @@
-tox
+tox >= 3.21.4

build/pkgs/tox/spkg-configure.m4

@@ -1,8 +1,6 @@
 SAGE_SPKG_CONFIGURE([tox], [
- dnl src/tox.ini has only minimal version requirements. We use 2.5.0 just to set a baseline.
- dnl (SAGE_ROOT/tox.ini needs negated factor conditions introduced in 3.0.0, but it is
- dnl best to run it with system tox anyway.)
- m4_pushdef([TOX_MIN_VERSION], [2.5.0])
+ dnl Use non-ancient tox with full support for PEP 517.
+ m4_pushdef([TOX_MIN_VERSION], [3.21.4])
  AC_CACHE_CHECK([for tox >= ]TOX_MIN_VERSION, [ac_cv_path_TOX], [
  AC_PATH_PROGS_FEATURE_CHECK([TOX], [tox], [
  tox_version=$($ac_path_TOX --version 2> /dev/null | tail -1)

pkgs/sagemath-categories/setup.cfg.m4

@@ -30,6 +30,7 @@ install_requires =
  esyscmd(`sage-get-system-packages install-requires \
  cython \
  pkgconfig \
+ ipython \
  gmpy2 \
  cysignals \
  | sed "2,\$s/^/ /;"')dnl

pkgs/sagemath-categories/tox.ini

@@ -7,33 +7,38 @@
 # pkgs/sagemath-categories/.tox/sagepython/bin/python
 #
 [tox]
+envlist =
+ sagepython-norequirements
 
 [testenv]
-deps = -rrequirements.txt
+deps =
+ !norequirements: -rrequirements.txt
 
 setenv =
  # Sage scripts such as sage-runtests like to use $HOME/.sage
  HOME={envdir}
 
 passenv =
- SAGE_NUM_THREADS
- SAGE_NUM_THREADS_PARALLEL
+ # Parallel build
+ SAGE_NUM_THREADS
+ SAGE_NUM_THREADS_PARALLEL
+ # SAGE_VENV only for referring to the basepython
+ sagepython: SAGE_VENV
 
 whitelist_externals =
  bash
 
-
 commands =
  # Beware of the treacherous non-src layout. "./sage/" shadows the install sage package.
- python -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all'
+ {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.cpython.builtin_types, sage.cpython.cython_metaclass, sage.cpython.debug, sage.structure.all, sage.categories.all'
 
  # Test that importing sage.categories.all initializes categories
- python -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()'
+ {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()'
 
  bash -c 'cd bin && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_categories --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"'
 
 [testenv:sagepython]
-passenv =
- SAGE_VENV
+basepython = {env:SAGE_VENV}/bin/python3
 
+[testenv:sagepython-norequirements]
 basepython = {env:SAGE_VENV}/bin/python3

pkgs/sagemath-environment/tox.ini

@@ -1,34 +1,41 @@
 # To build and test in the tox environment:
 #
-# ./sage -sh -c '(cd pkgs/sagemath-environment && tox -v -v)'
+# ./sage -sh -c '(cd pkgs/sagemath-environment && tox -v -v -e sagepython)'
 #
 # To test interactively:
 #
-# pkgs/sagemath-environment/.tox/python/bin/python
+# pkgs/sagemath-environment/.tox/sagepython/bin/python
 #
 [tox]
+envlist =
+ sagepython-norequirements
+
 isolated_build = True
 
 [testenv]
-deps = -rrequirements.txt
+deps =
+ !norequirements: -rrequirements.txt
 
 setenv =
  # Sage scripts such as sage-runtests like to use $HOME/.sage
  HOME={envdir}
 
 passenv =
- SAGE_NUM_THREADS
- SAGE_NUM_THREADS_PARALLEL
+ # Parallel build
+ SAGE_NUM_THREADS
+ SAGE_NUM_THREADS_PARALLEL
+ # SAGE_VENV only for referring to the basepython
+ sagepython: SAGE_VENV
 
 whitelist_externals =
  bash
 
 commands =
  # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package.
- python -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package'
+ {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.features.all import all_features; print(sorted(all_features(), key=lambda x: x.name)); import sage.misc.package'
 
 [testenv:sagepython]
-passenv =
- SAGE_VENV
+basepython = {env:SAGE_VENV}/bin/python3
 
+[testenv:sagepython-norequirements]
 basepython = {env:SAGE_VENV}/bin/python3

pkgs/sagemath-objects/setup.cfg.m4

@@ -30,6 +30,7 @@ install_requires =
  esyscmd(`sage-get-system-packages install-requires \
  cython \
  pkgconfig \
+ ipython \
  gmpy2 \
  cysignals \
  | sed "2,\$s/^/ /;"')dnl