sagemathgh-37430: Refactor bootstrap, bootstrap-conda, `m4/sage_s…

…pkg_collect.m4`, `sage-spkg-info` through `sage --package properties`, `sage-get-system-packages`

    
<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes sagemath#1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->

Using the command `sage --package properties` added in
sagemath#37018, we eliminate some direct
references to `build/pkgs/` from various scripts and reduce code
duplication in reading SPKG metadata.

Using other `sage --package` commands, we also simplify the code for
`bootstrap -s` and `bootstrap -D`.

Review instructions: Note that there are changes to the
`bootstrap`-generated file `m4/sage_spkg_configures.m4`:
- The macro `SAGE_SPKG_COLLECT_INIT` is now called explicitly, with an
argument (the full list of packages).
- Lines such as `m4_define([SPKG_INSTALL_REQUIRES_exceptiongroup],
[['exceptiongroup; python_version<\"3.11\"',]])dnl` are added for use by
the macro `SAGE_PYTHON_PACKAGE_CHECK` (replacing direct access to
`version_requirements.txt` files)

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes sagemath#12345". -->
This is:
- Split out from sagemath#36740
<!-- If your change requires a documentation PR, please link it
appropriately. -->

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- sagemath#12345: short description why this is a dependency
- sagemath#34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: sagemath#37430
Reported by: Matthias Köppe
Reviewer(s): Julian Rüth, Kwankyu Lee, Matthias Köppe, Tobias Diez

bootstrap

@@ -29,9 +29,7 @@ cd "$SAGE_ROOT"
 
 export PATH="$SAGE_ROOT/build/bin:$PATH"
 
-PKG=build/pkgs/configure
 MAKE="${MAKE:-make}"
-CONFVERSION=$(cat $PKG/package-version.txt)
 
 
 bootstrap () {
@@ -54,6 +52,7 @@ bootstrap () {
     for a in m4/sage_spkg_versions.m4 m4/sage_spkg_versions_toml.m4; do
         echo 'changequote(`>>>'"'"', `<<<'"')dnl" >> $a
     done
+    eval $(sage-package properties --format=shell :all:)
     spkg_configures=""
     # initialize SAGE_ENABLE... options for standard packages
     for pkgname in $(sage-package list :standard:); do
@@ -62,75 +61,67 @@ AS_VAR_SET_IF([SAGE_ENABLE_$pkgname], [], [AS_VAR_SET([SAGE_ENABLE_$pkgname], [y
     done
     # --enable-SPKG options
     for pkgname in $(sage-package list :optional: :experimental:); do
+        eval DIR=\$path_$pkgname pkgtype=\$type_$pkgname pkgsource=\$source_$pkgname
+        case "$pkgname:$pkgsource" in
+            *:pip)
                 # Issue #29629: Temporary solution for Sage 9.1: Do not provide
                 # --enable-SPKG options for installing pip packages
-                if [ ! -f build/pkgs/$pkgname/requirements.txt ]; then
-                    pkgtype="$(cat build/pkgs/$pkgname/type)"
-                    # Issue #29124: Do not provide --enable-_recommended and similar
-                    case "$pkgname" in
-                        _*) ;;
-                        *)  spkg_configures="$spkg_configures
+                ;;
+            _*:*)
+                # Issue #29124: Do not provide --enable-_recommended and similar
+                ;;
+            *:none)
+                # Issue #31163: Just an optional dummy package
+                spkg_configures="$spkg_configures
 AC_SUBST(SAGE_ENABLE_$pkgname, [if_installed])"
-                            if [ -f build/pkgs/$pkgname/spkg-install -o -f build/pkgs/$pkgname/spkg-install.in ]; then
-                                # Issue #31163: Not just an optional dummy package
-                                spkg_configures="$spkg_configures
-SAGE_SPKG_ENABLE([$pkgname], [$pkgtype], [$(grep -v ^= build/pkgs/$pkgname/SPKG.rst | head -n1 2>/dev/null || echo $pkgname)])"
-                            fi
-                            ;;
-                    esac
-                fi
+                ;;
+            *:*)
+                spkg_configures="$spkg_configures
+AC_SUBST(SAGE_ENABLE_$pkgname, [if_installed])"
+                spkg_configures="$spkg_configures
+SAGE_SPKG_ENABLE([$pkgname], [$pkgtype], [$(grep -v ^= "$DIR/SPKG.rst" | head -n1 2>/dev/null || echo $pkgname)])"
+                ;;
+        esac
     done
+    cat >> m4/sage_spkg_configures.m4 <<EOF
+SAGE_SPKG_COLLECT_INIT([$(echo $(sage-package list))])
+EOF
     for pkgname in $(sage-package list --has-file spkg-configure.m4); do
-        echo "m4_sinclude([build/pkgs/$pkgname/spkg-configure.m4])"
+        eval DIR=\$path_$pkgname
+        echo "m4_sinclude([$DIR/spkg-configure.m4])"
         config="SAGE_SPKG_CONFIGURE_$(echo ${pkgname} | tr '[a-z]' '[A-Z]')"
-        if grep -q SAGE_PYTHON_PACKAGE_CHECK build/pkgs/$pkgname/spkg-configure.m4; then
+        if grep -q SAGE_PYTHON_PACKAGE_CHECK "$DIR/spkg-configure.m4"; then
             spkg_configures_python="$spkg_configures_python
 $config"
         else
             spkg_configures="$spkg_configures
 $config"
         fi
     done >> m4/sage_spkg_configures.m4
-    cat >> m4/sage_spkg_configures.m4 <<EOF
-$spkg_configures
-$spkg_configures_python
-EOF
     for pkgname in $(sage-package list); do
-        DIR=build/pkgs/$pkgname
-        pkgtype="$(cat $DIR/type)"
-        if test -f "$DIR/requirements.txt"; then
-            SPKG_SOURCE=pip
-        elif test ! -f "$DIR/checksums.ini"; then
-            if test -f "$DIR/spkg-install" -o -f "$DIR/spkg-install.in"; then
-                SPKG_SOURCE=script
-            else
-                # a dummy package
-                SPKG_SOURCE=none
-            fi
-        else
-            SPKG_SOURCE=normal
-        fi
-        if test -f "$DIR/trees.txt"; then
-            SPKG_TREE_VAR="$(sed "s/#.*//;" "$DIR/trees.txt")"
-        else
-            SPKG_TREE_VAR=SAGE_LOCAL
-            if test -f "$DIR/requirements.txt" -o -f "$DIR/version_requirements.txt"; then
-                # A Python package
-                SPKG_TREE_VAR=SAGE_VENV
-                echo "define(>>>SPKG_INSTALL_REQUIRES_${pkgname}<<<, >>>$(echo $(sage-get-system-packages install-requires ${pkgname}))<<<)dnl" >> m4/sage_spkg_versions.m4
-                echo "define(>>>SPKG_INSTALL_REQUIRES_${pkgname}<<<, >>>$(echo $(sage-get-system-packages install-requires-toml ${pkgname}))<<<)dnl" >> m4/sage_spkg_versions_toml.m4
-            fi
+        eval DIR=\$path_$pkgname pkgtype=\$type_$pkgname SPKG_SOURCE=\$source_$pkgname SPKG_TREE_VAR=\$trees_$pkgname
+        if test -f "$DIR/requirements.txt" -o -f "$DIR/version_requirements.txt"; then
+            # A Python package
+            echo "define(>>>SPKG_INSTALL_REQUIRES_${pkgname}<<<, >>>$(echo $(sage-get-system-packages install-requires ${pkgname}))<<<)dnl" >> m4/sage_spkg_versions.m4
+            INSTALL_REQUIRES_TOML=
+            echo "define(>>>SPKG_INSTALL_REQUIRES_${pkgname}<<<, >>>$(echo $(sage-get-system-packages install-requires-toml ${pkgname}))<<<)dnl" >> m4/sage_spkg_versions_toml.m4
+            echo "m4_define([SPKG_INSTALL_REQUIRES_${pkgname}], [[$(echo $(sage-get-system-packages install-requires-toml ${pkgname} | sed 's/"/\\"/g'))]])dnl" >> m4/sage_spkg_configures.m4
         fi
         spkg_finalizes="$spkg_finalizes
 SAGE_SPKG_FINALIZE([$pkgname], [$pkgtype], [$SPKG_SOURCE], [$SPKG_TREE_VAR])"
     done
-    echo "$spkg_finalizes" >> m4/sage_spkg_configures.m4
     for a in m4/sage_spkg_versions.m4 m4/sage_spkg_versions_toml.m4; do
         echo 'changequote(>>>`<<<, >>>'"'"'<<<)dnl' >> $a
     done
+    cat >> m4/sage_spkg_configures.m4 <<EOF
+$spkg_configures
+$spkg_configures_python
+$spkg_finalizes
+EOF
 
     for pkgname in $(sage-package list --has-file bootstrap "$@"); do
-        (cd build/pkgs/$pkgname && if [ -x bootstrap ]; then ./bootstrap; else echo >&2 "bootstrap:$LINENO: Nothing to do for $pkgname"; fi) || exit 1
+        eval DIR=\$path_$pkgname
+        (cd "$DIR" && if [ -x bootstrap ]; then ./bootstrap; else echo >&2 "bootstrap:$LINENO: Nothing to do for $pkgname"; fi) || exit 1
     done
 
     if [ $# != 0 ]; then
@@ -196,10 +187,10 @@ SAGE_SPKG_FINALIZE([$pkgname], [$pkgtype], [$SPKG_SOURCE], [$SPKG_TREE_VAR])"
 bootstrap_download () {
     SAGE_DL_LOGLEVEL=""
     [ "${BOOTSTRAP_QUIET}" = "yes" ] && SAGE_DL_LOGLEVEL="--log=WARNING"
-    sage-download-file ${SAGE_DL_LOGLEVEL} configure-$CONFVERSION.tar.gz
 
+    CONFBALL=$(sage-package download $SAGE_DL_LOGLEVEL configure)
     if [ $? -ne 0 ]; then
-        echo >&2 "Error: downloading configure-$CONFVERSION.tar.gz failed"
+        echo >&2 "Error: downloading configure tarball failed"
         exit 1
     fi
 
@@ -255,16 +246,8 @@ save () {
 	build/pkgs/setuptools/version_requirements.txt \
 	build/pkgs/wheel/version_requirements.txt
 
-    # Update version
-    echo "$NEWCONFVERSION" >$PKG/package-version.txt
-
-    # Compute checksum
-    if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
-	sage-package fix-checksum configure
-    else
-	# Hide the "Updating checksum..." message
-	sage-package fix-checksum configure > /dev/null
-    fi
+    # Update version; this re-computes the checksum
+    sage-package update configure "$NEWCONFVERSION"
 }
 
 
@@ -304,7 +287,6 @@ do
 done
 shift $(($OPTIND - 1))
 export BOOTSTRAP_QUIET
-CONFBALL="upstream/configure-$CONFVERSION.tar.gz"
 
 if [ $DOWNLOAD$SAVE = yesyes ]; then
     echo >&2 "$0: refusing to download and save."

bootstrap-conda

@@ -18,10 +18,11 @@ SAGELIB_PACKAGES=
 SAGELIB_OPTIONAL_PACKAGES=
 DEVELOP_PACKAGES=
 
+eval $(sage-package properties --format=shell :all:)
+
 for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sagemath); do
-    PKG_SCRIPTS=build/pkgs/$PKG_BASE
+    eval PKG_SCRIPTS=\$path_$PKG_BASE PKG_TYPE=\$type_$PKG_BASE
     SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt
-    PKG_TYPE=$(cat $PKG_SCRIPTS/type)
     PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
     if [ -n "$PKG_SYSTEM_PACKAGES" ]; then
         if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
@@ -133,12 +134,11 @@ echo >&2 $0:$LINENO: generate conda environment files
         echo >&4 "  - pip:"
         echo >&5 "  - pip:"
         for PKG_BASE in $(sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file version_requirements.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src); do
-            PKG_SCRIPTS=build/pkgs/$PKG_BASE
+            eval PKG_SCRIPTS=\$path_$PKG_BASE PKG_TYPE=\$type_$PKG_BASE
             SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt
             if [ ! -f $SYSTEM_PACKAGES_FILE ]; then
                 SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/version_requirements.txt
             fi
-            PKG_TYPE=$(cat $PKG_SCRIPTS/type)
             if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then
                 : # cannot install packages that depend on the Sage library
             else

build/bin/sage-get-system-packages

@@ -15,26 +15,40 @@ fi
 case "$SYSTEM" in
     install-requires)
         # Collect from src/pyproject.toml or from version_requirements.txt (falling back to requirements.txt) and output it in the format
-        # needed by setup.cfg [options] version_requirements=
-        SYSTEM_PACKAGES_FILE_NAMES="version_requirements.txt requirements.txt"
-        STRIP_COMMENTS="sed s/#.*//;/^[[:space:]]*$/d;"
-        FROM_PYPROJECT_TOML=1
+        # needed by setup.cfg [options] install_requires=
+        SYSTEM_PACKAGES_FILE_NAMES="src/pyproject.toml version_requirements.txt requirements.txt"
+        # also normalizes quotes from "" to ''.
+        STRIP_COMMENTS="sed s/#.*//;/^[[:space:]]*$/d;s/\"/'/g;"
         COLLECT=
         ;;
     install-requires-toml)
         # Collect from src/pyproject.toml or from version_requirements.txt (falling back to requirements.txt) and output it in the format
         # needed by pyproject.toml [build-system] requires=
-        SYSTEM_PACKAGES_FILE_NAMES="version_requirements.txt requirements.txt"
-        STRIP_COMMENTS="sed s/#.*//;/^[[:space:]]*$/d;s/^/'/;s/$/',/;"
-        FROM_PYPROJECT_TOML=1
+        SYSTEM_PACKAGES_FILE_NAMES="src/pyproject.toml version_requirements.txt requirements.txt"
+        # also normalizes quotes from '' to "".
+        STRIP_COMMENTS="sed s/#.*//;/^[[:space:]]*$/d;s/'/\"/g;s/^/'/;s/$/',/;"
         COLLECT=
         ;;
     pip)
-        SYSTEM_PACKAGES_FILE_NAMES="requirements.txt version_requirements.txt"
+        SYSTEM_PACKAGES_FILE_NAMES="requirements.txt src/pyproject.toml version_requirements.txt"
         STRIP_COMMENTS='sed s/#.*//;s/[[:space:]]//g;'
-        FROM_PYPROJECT_TOML=1
         COLLECT=echo
         ;;
+    versions)
+        # For use in sage-spkg-info
+        SYSTEM_PACKAGES_FILE_NAMES="package-version.txt requirements.txt src/pyproject.toml version_requirements.txt"
+        strip_comments () {
+            TEXT=$(sed "s/#.*//;/^[[:space:]]*$/d;s/\"/'/g;s/^/    /;" "$@")
+            if [ -n "$(echo $TEXT)" ]; then
+                echo "$NAME::"
+                echo
+                echo "$TEXT"
+                echo
+            fi
+        }
+        STRIP_COMMENTS=strip_comments
+        COLLECT=
+        ;;
     *)
         if [ "$SYSTEM" = auto ]; then
             SYSTEM=$(sage-guess-package-system 2>/dev/null)
@@ -45,7 +59,6 @@ case "$SYSTEM" in
         fi
         SYSTEM_PACKAGES_FILE_NAMES="distros/$SYSTEM.txt"
         STRIP_COMMENTS="sed s/#.*//;s/\${PYTHON_MINOR}/${PYTHON_MINOR}/g"
-        FROM_PYPROJECT_TOML=0
         COLLECT=echo
         ;;
 esac
@@ -57,19 +70,10 @@ case "$SPKGS" in
 esac
 
 for PKG_BASE in $SPKGS; do
-    if [ $FROM_PYPROJECT_TOML -eq 1 ]; then
-        if [ -f "$SAGE_ROOT/src/pyproject.toml" ]; then
-            # Extract from the "requires" block in src/pyproject.toml
-            # Packages are in the format "'sage-conf ~= 10.3b3',"
-            PACKAGE_INFO=$(sed -n '/requires *= *\[/,/^\]/s/^ *'\''\('$PKG_BASE'.*\)'\'',/\1/p' "$SAGE_ROOT/src/pyproject.toml")
-            if [ -n "$PACKAGE_INFO" ]; then
-                echo "$PACKAGE_INFO" | ${STRIP_COMMENTS}
-                continue
-            fi
-        fi
-    fi
-
     case "$SYSTEM:$ENABLE_SYSTEM_SITE_PACKAGES" in
+        versions*)
+            # Show everything.
+            ;;
         install-requires*|pip*)
             # This is output for installation of packages into a Python environment.
             # So it's OK to use any Python packages.
@@ -93,14 +97,30 @@ for PKG_BASE in $SPKGS; do
     esac
 
     for NAME in $SYSTEM_PACKAGES_FILE_NAMES; do
-        SYSTEM_PACKAGES_FILE="$SAGE_ROOT"/build/pkgs/$PKG_BASE/$NAME
-        if [ -f $SYSTEM_PACKAGES_FILE ]; then
-            if [ -z "$COLLECT" ]; then
-                ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE
-            else
-                $COLLECT $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE)
-            fi
-            break
-        fi
+        case $NAME in
+            *pyproject.toml)
+                SYSTEM_PACKAGES_FILE="$SAGE_ROOT"/$NAME
+                if [ -f "$SYSTEM_PACKAGES_FILE" ]; then
+                    # Extract from the "requires" block in src/pyproject.toml
+                    # Packages are in the format "'sage-conf ~= 10.3b3',"
+                    PACKAGE_INFO=$(sed -n '/requires *= *\[/,/^\]/s/^ *'\''\('$PKG_BASE'.*\)'\'',/\1/p' "$SAGE_ROOT/src/pyproject.toml")
+                    if [ -n "$PACKAGE_INFO" ]; then
+                        echo "$PACKAGE_INFO" | ${STRIP_COMMENTS}
+                        [ $SYSTEM = versions ] || break
+                    fi
+                fi
+                ;;
+            *)
+                SYSTEM_PACKAGES_FILE="$SAGE_ROOT"/build/pkgs/$PKG_BASE/$NAME
+                if [ -f $SYSTEM_PACKAGES_FILE ]; then
+                    if [ -z "$COLLECT" ]; then
+                        ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE
+                    else
+                        $COLLECT $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE)
+                    fi
+                    [ $SYSTEM = versions ] || break
+                fi
+                ;;
+        esac
     done
 done

build/bin/sage-spkg-info

@@ -54,14 +54,7 @@ echo
 echo "Version Information"
 echo "-------------------"
 echo
-for a in package-version.txt requirements.txt version_requirements.txt; do
-    if [ -f "$PKG_SCRIPTS"/"$a" ]; then
-        echo "$a::"
-        echo
-        sed 's/^/    /' "$PKG_SCRIPTS/$a"
-        echo
-    fi
-done
+sage-get-system-packages versions $PKG_BASE
 echo
 echo "Equivalent System Packages"
 echo "--------------------------"

m4/sage_python_package_check.m4

@@ -8,8 +8,8 @@
 #   Determine if the system copy of a python package can be used by sage.
 #
 #   This macro uses setuptools.version's pkg_resources to check that the
-#   "version_requirements.txt" file for the named package is satisfied, and
-#   it can typically fail in four ways:
+#   "version_requirements.txt" file (or entry in "src/pyproject.toml") for
+#   the named package is satisfied, and it can typically fail in four ways:
 #
 #     1. If --enable-system-site-packages was not passed to ./configure,
 #
@@ -19,8 +19,9 @@
 #
 #     4. If setuptools is not available to the system python,
 #
-#     5. If the contents of version_requirements.txt are not met (wrong
-#        version, no version, etc.) by the system python.
+#     5. If the contents of version_requirements.txt (or entry in
+#        "src/pyproject.toml") are not met (wrong version, no version,
+#        etc.) by the system python.
 #
 #   In any of those cases, we set sage_spkg_install_$package to "yes"
 #   so that the corresponding SPKG is installed. Otherwise, we do
@@ -56,29 +57,17 @@ AC_DEFUN([SAGE_PYTHON_PACKAGE_CHECK], [
                                   config.venv            dnl
                                   2>&AS_MESSAGE_LOG_FD], [
       AC_MSG_RESULT(yes)
-      dnl strip all comments from version_requirements.txt; this should leave
-      dnl only a single line containing the version specification for this
-      dnl package. Afterwards, convert all double-quotes to single quotes.
-      dnl Both work, but only single quotes are documented. However, at the
-      dnl time of writing, double quotes are more compatible with our toml
-      dnl generation in ./bootstrap. Converting them from double- to single-
-      dnl quotes on-the-fly here lets us support both (in this macro, at
-      dnl least).
-      SAGE_PKG_VERSPEC=$(sed                   \
-        -e '/^#/d'                             \
-        -e "s/\"/'/g"                          \
-        "./build/pkgs/$1/version_requirements.txt"
-      )
-      AC_MSG_CHECKING([for python package $1 ("${SAGE_PKG_VERSPEC}")])
+      dnl SAGE_PKG_VERSPEC is in the format of a toml list, but
+      dnl without surrounding brackets, of single-quoted strings,
+      dnl with any double-quotes escaped by backslash.
+      AS_VAR_SET([SAGE_PKG_VERSPEC], ["SPKG_INSTALL_REQUIRES_]$1["])
+      AC_MSG_CHECKING([for python package $1 (${SAGE_PKG_VERSPEC%,})])
 
       WITH_SAGE_PYTHONUSERBASE([dnl
-        dnl double-quote SAGE_PKG_VERSPEC because platform-specific
-        dnl dependencies like python_version<'3.11' will have single
-        dnl quotes in them. (We normalized the quotes earlier with sed.)
         AS_IF(
-          [config.venv/bin/python3 -c   dnl
-             "import pkg_resources;                                        dnl
-              pkg_resources.require(\"${SAGE_PKG_VERSPEC}\".splitlines())" dnl
+          [config.venv/bin/python3 -c dnl
+             "import pkg_resources; dnl
+              pkg_resources.require((${SAGE_PKG_VERSPEC}))" dnl
            2>&AS_MESSAGE_LOG_FD],
           [AC_MSG_RESULT(yes)],
           [AC_MSG_RESULT(no); sage_spkg_install_$1=yes]