gh-37434: Document building from the monorepo using `PIP_CONSTRAINT=$…

…SAGE_ROOT/constraints_pkgs.txt`

    
<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes #1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->
This is aimed at the intermediate level, for users who wish to build a
set of distribution packages as supplied in `pkgs/` but without using
the wheelhouse infrastructure of Sage-the-distribution.

Documentation included in the files.

It can also be tested using new tox environments defined in
`pkgs/sagemath-standard/tox.ini`, and is tested on GH Actions:
https://github.com/mkoeppe/sage/actions/workflows/ci-linux-
incremental.yml?query=branch%3Aconstraints_pkgs
- this also provides the missing automatic test for the **sagemath-
standard** distribution built out of `pkgs/sagemath-standard/`

Marked as critical because it is hoped to ease the adoption of
modularized distribution packages by downstream packagers.

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes #12345". -->
<!-- If your change requires a documentation PR, please link it
appropriately. -->

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- #12345: short description why this is a dependency
- #34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->

- Depends on #38515 (merged here)
- Depends on #38500 (merged here)
    
URL: #37434
Reported by: Matthias Köppe
Reviewer(s): François Bissey, Kwankyu Lee, Matthias Köppe

.github/workflows/ci-linux-incremental.yml

@@ -60,9 +60,9 @@ jobs:
           uninstall_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))
           build_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))
           if [ -n "$uninstall_targets" ]; then
-              echo "build_targets=$uninstall_targets reconfigure $build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
+              echo "build_targets=$uninstall_targets reconfigure $build_targets" >> $GITHUB_OUTPUT
           else
-              echo "build_targets=$build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
+              echo "build_targets=$build_targets" >> $GITHUB_OUTPUT
           fi
           cat $GITHUB_OUTPUT
       - uses: actions/checkout@v4
@@ -92,7 +92,7 @@ jobs:
       from_docker_target: "with-targets"
       from_docker_tag: "dev"
       docker_targets: "with-targets"
-      targets: "${{needs.changed_files.outputs.build_targets}} doc-html ptest-nodoc"
+      targets: "${{needs.changed_files.outputs.build_targets}} ci-build-with-fallback doc-html ptest-nodoc"
       tox_system_factors: >-
         ["ubuntu-focal",
          "ubuntu-noble",
@@ -108,6 +108,29 @@ jobs:
       docker_push_repository: ghcr.io/${{ github.repository }}/
       max_parallel: 8
 
+  constraints_pkgs-norequirements:
+    needs: [changed_files]
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets-pre"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets-pre"
+      targets_pre: "${{needs.changed_files.outputs.build_targets}} all-sage-local python3-ensure tox-ensure sagelib-tox-sagepython-constraints_pkgs-norequirements"
+      tox_system_factors: >-
+        ["ubuntu-focal",
+         "ubuntu-noble",
+         "debian-bookworm",
+         "fedora-40",
+         "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["standard"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+      max_parallel: 16
+
   site:
     needs: [changed_files]
     uses: ./.github/workflows/docker.yml

.github/workflows/ci-linux.yml

@@ -90,6 +90,22 @@ jobs:
       # and 'minimal-pre' below
       max_parallel: 20
 
+  standard-constraints_pkgs-norequirements:
+    if: ${{ success() || failure() }}
+    needs: [standard-pre]
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from previous stage (pre)
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/${{ github.repository }}/
+      from_docker_target: "with-targets-pre"
+      docker_targets: "with-targets-pre"
+      targets_pre: all-sage-local python3-ensure tox-ensure sagelib-tox-sagepython-constraints_pkgs-norequirements
+      tox_packages_factors: >-
+          ["standard"]
+      max_parallel: 15
+
   standard-sitepackages:
     if: ${{ success() || failure() }}
     needs: [standard-pre]

build/make/Makefile.in

@@ -779,12 +779,14 @@ $(1)-sdist: FORCE python_build sage_setup cython
 		. '$$(SAGE_ROOT)/src/bin/sage-env' && \
 		'$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-src'
 
-# Recursive tox invocation (note - we do not set the environment here).
+# Recursive tox invocation
 # Setting SAGE_SPKG_WHEELS is for the benefit of sagelib's tox.ini
 $(1)-tox-%: FORCE
 	$(AM_V_at)cd '$$(SAGE_ROOT)/build/pkgs/$(1)/src' && \
-		export PATH="$$(SAGE_ORIG_PATH)" && \
-		SAGE_SPKG_WHEELS=$$(SAGE_LOCAL)/var/lib/sage/wheels \
+		. '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \
+		. '$$(SAGE_ROOT)/src/bin/sage-env-config' && \
+		. '$$(SAGE_ROOT)/src/bin/sage-env' && \
+		SAGE_SPKG_WHEELS=$$(SAGE_VENV)/var/lib/sage/wheels \
 		tox -v -v -v -e $$*
 
 .PHONY: $(1) $(1)-uninstall $(1)-clean $(1)-build-deps $(1)-no-deps $(1)-clean

constraints_pkgs.txt

@@ -0,0 +1,39 @@
+# This "constraints file" can be used for forcing pip
+# (and any tools that delegate to pip, such as pypa/build)
+# to install the distribution packages included in
+# the SageMath monorepository only from their source trees
+# in SAGE_ROOT/pkgs/ instead of from PyPI.
+#
+# Example: Building a sagemath-standard wheel
+#
+# [alice@localhost sage]$ ./bootstrap
+# [alice@localhost sage]$ ./configure
+# [alice@localhost sage]$ export MAKE="make -j16" SAGE_NUM_THREADS=16
+# [alice@localhost sage]$ make all-sage-local
+# [alice@localhost sage]$ export PIP_CONSTRAINT="$(pwd)/constraints_pkgs.txt"
+# [alice@localhost sage]$ ./sage -sh -c 'python3 -m build -v -v pkgs/sagemath-standard'
+#
+# Non-example: Installing the built wheel using the same
+# constraints file will fail because sagemath-standard is one
+# of the distribution packages listed below. It will conflict
+# with the built wheel for sagemath-standard!
+# Use "pkgs/sagemath-standard/constraints_pkgs.txt" instead.
+
+# Reference on the format:
+# https://pip.pypa.io/en/stable/user_guide/#constraints-files
+#
+sage_conf            @ file://${SAGE_ROOT}/pkgs/sage-conf
+sage_docbuild        @ file://${SAGE_ROOT}/pkgs/sage-docbuild
+sage_setup           @ file://${SAGE_ROOT}/pkgs/sage-setup
+sage_sws2rst         @ file://${SAGE_ROOT}/pkgs/sage-sws2rst
+sagemath-bliss       @ file://${SAGE_ROOT}/pkgs/sagemath-bliss
+sagemath-categories  @ file://${SAGE_ROOT}/pkgs/sagemath-categories
+sagemath-coxeter3    @ file://${SAGE_ROOT}/pkgs/sagemath-coxeter3
+sagemath-environment @ file://${SAGE_ROOT}/pkgs/sagemath-environment
+sagemath-mcqd        @ file://${SAGE_ROOT}/pkgs/sagemath-mcqd
+sagemath-meataxe     @ file://${SAGE_ROOT}/pkgs/sagemath-meataxe
+sagemath-objects     @ file://${SAGE_ROOT}/pkgs/sagemath-objects
+sagemath-repl        @ file://${SAGE_ROOT}/pkgs/sagemath-repl
+sagemath-sirocco     @ file://${SAGE_ROOT}/pkgs/sagemath-sirocco
+sagemath-standard    @ file://${SAGE_ROOT}/pkgs/sagemath-standard
+sagemath-tdlib       @ file://${SAGE_ROOT}/pkgs/sagemath-tdlib

pkgs/sagemath-standard/README.rst

@@ -8,24 +8,26 @@ About SageMath
    "Creating a Viable Open Source Alternative to
     Magma, Maple, Mathematica, and MATLAB"
 
-   Copyright (C) 2005-2020 The Sage Development Team
+   Copyright (C) 2005-2024 The Sage Development Team
 
    https://www.sagemath.org
 
 SageMath fully supports all major Linux distributions, recent versions of macOS, and Windows (Windows Subsystem for Linux).
 
-The traditional and recommended way to install SageMath is from source via Sage-the-distribution (https://www.sagemath.org/download-source.html).  Sage-the-distribution first builds a large number of open source packages from source (unless it finds suitable versions installed in the system) and then installs the Sage Library (sagelib, implemented in Python and Cython).
+See https://doc.sagemath.org/html/en/installation/index.html
+for general installation instructions.
 
 
-About this experimental pip-installable source distribution
------------------------------------------------------------
+About this pip-installable distribution package
+-----------------------------------------------
 
-This pip-installable source distribution `sagemath-standard` is an experimental distribution of the Sage Library.  Use at your own risk.
+This pip-installable source distribution `sagemath-standard` is a
+distribution of the Sage Library.
 
 Building `sagemath-standard` has a large number of system packages as prerequisites.
 See https://doc.sagemath.org/html/en/installation/source.html#linux-recommended-installation
 for partial lists for various systems.
 
-The connection to the system environment is facilitated through the https://pypi.org/project/sage-conf/ distribution package.
+The connection to the system environment is facilitated through the https://pypi.org/project/sage-conf/ distribution package; for step-by-step installation instructions, see https://github.com/sagemath/sage/blob/develop/README.md#alternative-installation-using-pypi
 
 A modularization effort is in progress with the goal of making it possible to install parts of the Sage Library with fewer prerequisites. https://github.com/sagemath/sage/issues/29705

pkgs/sagemath-standard/constraints_pkgs.txt

@@ -0,0 +1,21 @@
+# This "constraints file" can be used for forcing pip
+# (and any tools that delegate to pip, such as pypa/build)
+# to install the dependencies of sagemath-standard that are
+# distribution packages included in the SageMath monorepository
+# only from their source trees in SAGE_ROOT/pkgs/
+# instead of from PyPI.
+#
+# Example:
+#
+# [alice@localhost sage]$ ./bootstrap
+# [alice@localhost sage]$ ./configure
+# [alice@localhost sage]$ export MAKE="make -j16" SAGE_NUM_THREADS=16
+# [alice@localhost sage]$ make all-sage-local
+# [alice@localhost sage]$ export PIP_CONSTRAINT="$(pwd)/pkgs/sagemath-standard/constraints_pkgs.txt"
+# [alice@localhost sage]$ ./sage -sh -c 'python3 -m build -v -v pkgs/sagemath-standard'
+#
+# Reference on the format:
+# https://pip.pypa.io/en/stable/user_guide/#constraints-files
+#
+sage_conf            @ file://${SAGE_ROOT}/pkgs/sage-conf
+sage_setup           @ file://${SAGE_ROOT}/pkgs/sage-setup

pkgs/sagemath-standard/tox.ini

@@ -31,6 +31,19 @@ envlist =
     #
     #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-nopypi)'
     #
+    # Build and test without using the concrete dependencies specified by requirements.txt,
+    # using the dependencies declared in pyproject.toml and setup.cfg (install-requires) only:
+    # Install the distribution packages included in the SageMath monorepository only from
+    # their source trees in SAGE_ROOT/pkgs/ (not from PyPI).
+    #
+    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-constraints_pkgs-norequirements)'
+    #
+    # Build dependencies according to requirements.txt (all versions fixed).
+    # Install the distribution packages included in the SageMath monorepository only from
+    # their source trees in SAGE_ROOT/pkgs/ (not from PyPI).
+    #
+    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-constraints_pkgs)'
+    #
     # EXPERIMENTAL ENVIRONMENTS:
     #
     # Build dependencies according to requirements.txt (all versions fixed).
@@ -80,6 +93,8 @@ passenv =
                              MAKEFLAGS
     # SAGE_VENV only for referring to the basepython or finding the wheels
     sagepython, sagewheels:  SAGE_VENV
+    # Used as an environment variable in constraints_pkgs.txt
+    constraints_pkgs:        SAGE_ROOT
     # Location of the wheels
     sagewheels:              SAGE_SPKG_WHEELS
 
@@ -88,6 +103,7 @@ setenv =
     # apply both to the installation of the dependencies and of the package
     sagewheels:              PIP_FIND_LINKS=file://{env:SAGE_SPKG_WHEELS:{env:SAGE_VENV:{toxinidir}/../../../../venv}/var/lib/sage/wheels}
     nopypi:                  PIP_NO_INDEX=true
+    constraints_pkgs:        PIP_CONSTRAINT={toxinidir}/constraints_pkgs.txt
     # No build isolation for PEP 517 packages - use what is already in the environment
     # Note that this pip env "NO" variable uses inverted logic:
     # PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
@@ -153,6 +169,17 @@ setenv =                     {[pkgenv]setenv}
 
 basepython = {env:SAGE_VENV}/bin/python3
 
+[testenv:.pkg-sagepython-constraints_pkgs]
+passenv =                    {[pkgenv]passenv}
+                             SAGE_VENV
+    # Location of constraints_pkgs.txt
+                             SAGE_ROOT
+
+setenv =                     {[pkgenv]setenv}
+                             PIP_CONSTRAINT={env:SAGE_ROOT}/constraints_pkgs.txt
+
+basepython =                 {env:SAGE_VENV}/bin/python3
+
 [testenv:sagepython]
 basepython = {env:SAGE_VENV}/bin/python3
 package_env = .pkg-sagepython
@@ -177,6 +204,14 @@ package_env = .pkg-sagepython
 basepython = {env:SAGE_VENV}/bin/python3
 package_env = .pkg-sagepython
 
+[testenv:sagepython-constraints_pkgs]
+basepython = {env:SAGE_VENV}/bin/python3
+package_env = .pkg-sagepython-constraints_pkgs
+
+[testenv:sagepython-constraints_pkgs-norequirements]
+basepython = {env:SAGE_VENV}/bin/python3
+package_env = .pkg-sagepython-constraints_pkgs
+
 
 [testenv:sagepython-sagewheels-nopypi-norequirements]
 basepython = {env:SAGE_VENV}/bin/python3

src/pyproject.toml

@@ -14,6 +14,7 @@ requires = [
     'memory_allocator',
     'numpy >=1.19',
     'pkgconfig',
+    'jinja2',
 ]
 build-backend = "setuptools.build_meta"
 

src/sage/matroids/basis_exchange_matroid.pxd

@@ -46,19 +46,19 @@ cdef class BasisExchangeMatroid(Matroid):
     cpdef _move_current_basis(self, X, Y)
 
     cpdef frozenset _max_independent(self, frozenset F)
-    cpdef int _rank(self, frozenset F)
+    cpdef int _rank(self, frozenset F) except -1
     cpdef frozenset _circuit(self, frozenset F)
     cpdef frozenset _fundamental_circuit(self, frozenset B, e)
     cpdef frozenset _closure(self, frozenset F)
 
     cpdef frozenset _max_coindependent(self, frozenset F)
-    cpdef int _corank(self, frozenset F)
+    cpdef int _corank(self, frozenset F) noexcept
     cpdef frozenset _cocircuit(self, frozenset F)
     cpdef frozenset _fundamental_cocircuit(self, frozenset B, e)
     cpdef frozenset _coclosure(self, frozenset F)
 
     cpdef frozenset _augment(self, frozenset X, frozenset Y)
-    cpdef bint _is_independent(self, frozenset F)
+    cpdef bint _is_independent(self, frozenset F) noexcept
 
     cpdef list whitney_numbers2(self)
     cdef  _whitney_numbers2_rec(self, object f_vec, bitset_t* flats, bitset_t* todo, long elt, long rnk)
@@ -90,6 +90,6 @@ cdef class BasisExchangeMatroid(Matroid):
     cpdef _is_isomorphism(self, other, morphism)
     cdef bint __is_isomorphism(self, BasisExchangeMatroid other, morphism) noexcept
 
-    cpdef bint is_valid(self)
+    cpdef bint is_valid(self) noexcept
 
 cdef bint nxksrd(bitset_s *b, long n, long k, bint succ) noexcept

src/sage/matroids/basis_exchange_matroid.pyx

@@ -652,7 +652,7 @@ cdef class BasisExchangeMatroid(Matroid):
         self.__max_independent(self._output, self._input)
         return self.__unpack(self._output)
 
-    cpdef int _rank(self, frozenset F):
+    cpdef int _rank(self, frozenset F) except -1:
         """
         Compute the rank of a subset of the groundset.
 
@@ -796,7 +796,7 @@ cdef class BasisExchangeMatroid(Matroid):
         self.__max_coindependent(self._output, self._input)
         return self.__unpack(self._output)
 
-    cpdef int _corank(self, frozenset F):
+    cpdef int _corank(self, frozenset F) noexcept:
         """
         Return the corank of a set.
 
@@ -940,7 +940,7 @@ cdef class BasisExchangeMatroid(Matroid):
         self.__augment(self._output, self._input, self._input2)
         return self.__unpack(self._output)
 
-    cpdef bint _is_independent(self, frozenset F):
+    cpdef bint _is_independent(self, frozenset F) noexcept:
         """
         Test if input is independent.
 
@@ -2231,7 +2231,7 @@ cdef class BasisExchangeMatroid(Matroid):
 
         return self._characteristic_setsystem()._isomorphism(other._characteristic_setsystem(), PS, PO) is not None
 
-    cpdef bint is_valid(self):
+    cpdef bint is_valid(self) noexcept:
         r"""
         Test if the data obey the matroid axioms.
 

src/sage/matroids/basis_matroid.pxd

@@ -15,7 +15,7 @@ cdef class BasisMatroid(BasisExchangeMatroid):
 
     cdef reset_current_basis(self)
 
-    cpdef bint _is_basis(self, frozenset X)
+    cpdef bint _is_basis(self, frozenset X) noexcept
 
     cpdef bases_count(self)
     cpdef SetSystem bases(self)

src/sage/matroids/basis_matroid.pyx

@@ -298,7 +298,7 @@ cdef class BasisMatroid(BasisExchangeMatroid):
 
     # a function that is very efficient for this class
 
-    cpdef bint _is_basis(self, frozenset X):
+    cpdef bint _is_basis(self, frozenset X) noexcept:
         """
         Test if input is a basis.
 

src/sage/matroids/circuit_closures_matroid.pxd

@@ -5,9 +5,9 @@ cdef class CircuitClosuresMatroid(Matroid):
     cdef dict _circuit_closures  # _CC
     cdef int _matroid_rank  # _R
     cpdef frozenset groundset(self)
-    cpdef int _rank(self, frozenset X)
+    cpdef int _rank(self, frozenset X) except -1
     cpdef full_rank(self)
-    cpdef bint _is_independent(self, frozenset F)
+    cpdef bint _is_independent(self, frozenset F) noexcept
     cpdef frozenset _max_independent(self, frozenset F)
     cpdef frozenset _circuit(self, frozenset F)
     cpdef dict circuit_closures(self)

src/sage/matroids/circuit_closures_matroid.pyx

@@ -179,7 +179,7 @@ cdef class CircuitClosuresMatroid(Matroid):
         """
         return frozenset(self._groundset)
 
-    cpdef int _rank(self, frozenset X):
+    cpdef int _rank(self, frozenset X) except -1:
         """
         Return the rank of a set ``X``.
 
@@ -221,7 +221,7 @@ cdef class CircuitClosuresMatroid(Matroid):
         """
         return self._matroid_rank
 
-    cpdef bint _is_independent(self, frozenset F):
+    cpdef bint _is_independent(self, frozenset F) noexcept:
         """
         Test if input is independent.