gh-36747: build/bin/sage-spkg: Add support for installing script pa…

…ckages

    
<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes #1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->
Split out and updated from #29386.

The section of `build/make/Makefile.in` that deals with script packages
can be consolidated with the section on normal packages later, in a
follow-up PR after #36740, #36738.

We switch the `spkg-install` scripts of the `sage*` script packages to
the templated versions (`spkg-install.in`), which is a simplification.
`spkg-check` remains as is because we still invoke it directly (changing
this will need #36738).

Resolves #29386.

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes #12345". -->
<!-- If your change requires a documentation PR, please link it
appropriately. -->

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- #12345: short description why this is a dependency
- #34567: ...
-->
- Depends on #36737 (for testing)

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: #36747
Reported by: Matthias Köppe
Reviewer(s): Michael Orlitzky

bootstrap

@@ -92,7 +92,7 @@ EOF
         if test -f "$DIR/requirements.txt"; then
             SPKG_SOURCE=pip
         elif test ! -f "$DIR/checksums.ini"; then
-            if test -f "$DIR/spkg-install"; then
+            if test -f "$DIR/spkg-install" -o -f "$DIR/spkg-install.in"; then
                 SPKG_SOURCE=script
             else
                 # a dummy package

build/bin/sage-spkg

@@ -323,21 +323,26 @@ export SAGE_SPKG_SCRIPTS="$SAGE_INST_LOCAL/var/lib/sage/scripts"
 export SAGE_SPKG_WHEELS="$SAGE_INST_LOCAL/var/lib/sage/wheels"
 
 # PKG_SRC should look like "package-VERSION" or just "package".
-# If VERSION matches the version in build/pkgs or there is no version
-# specified, use the local scripts; otherwise we try to find a package
-# in upstream.
+# VERSION, if provided, must match the version in build/pkgs.
 PKG_VER="${PKG_NAME#${PKG_BASE}}"
 PKG_VER="${PKG_VER#-}"
 PKG_SCRIPTS="$SAGE_ROOT/build/pkgs/$PKG_BASE"
-LOCAL_PKG_VER=`cat $PKG_SCRIPTS/package-version.txt 2>/dev/null`
+LOCAL_PKG_VER=`cat $PKG_SCRIPTS/package-version.txt 2>/dev/null || echo none`
 PKG_VER="$LOCAL_PKG_VER"
+if [ -n "$PKG_VER" -a "$PKG_VER" != "$LOCAL_PKG_VER" ]; then
+    echo >&2 "Error: Selecting a different version of a package is no longer supported"
+    exit 1
+fi
 if [ -z "$PKG_VER" ]; then
     PKG_NAME="${PKG_BASE}"
 else
     PKG_NAME="${PKG_BASE}-${PKG_VER}"
 fi
 PKG_BASE_VER=`echo $PKG_VER | sed 's/\.p[0-9][0-9]*$//'`
-PKG_NAME_UPSTREAM=`lookup_param tarball "$PKG_SCRIPTS/checksums.ini" | sed "s/VERSION/$PKG_BASE_VER/"`
+if [ -f "$PKG_SCRIPTS/checksums.ini" ]; then
+    # Normal/wheel package
+    PKG_NAME_UPSTREAM=`lookup_param tarball "$PKG_SCRIPTS/checksums.ini" | sed "s/VERSION/$PKG_BASE_VER/"`
+fi
 
 # Set the $SAGE_DESTDIR variable to be passed to the spkg-install
 # script (the script itself could set this, but better to standardize
@@ -356,7 +361,7 @@ WRAPPED_SCRIPTS="build install check preinst postinst $INSTALLED_SCRIPTS"
 
 warning_for_experimental_packages() { ############################
 if [ x`cat "$PKG_SCRIPTS/type"` = x"experimental" ]; then
-    if [ $YES != 1 ]; then
+    if [ $YES != 1 -a -n "$PKG_NAME_UPSTREAM" ]; then
         # We use /dev/tty here because our output may be redirected
         # to a logfile, or line-buffered.
         write_to_tty <<EOF
@@ -388,36 +393,31 @@ fi
 } ############################## warning_for_experimental_packages
 
 ensure_pkg_src() { ###############################################
-# If we haven't found the package yet, we must download it
 if [ ! -f "$PKG_SRC" ]; then
     if [ -n "$PKG_NAME_UPSTREAM" ]; then
-        # This is the new-style package way of downloading the tarball
+        # Normal or wheel package
         if ! sage-download-file $SAGE_DOWNLOAD_FILE_OPTIONS "$PKG_NAME_UPSTREAM"; then
             error_msg "Error downloading $PKG_NAME_UPSTREAM"
             exit 1
         fi
         PKG_SRC="$SAGE_DISTFILES/$PKG_NAME_UPSTREAM"
-    else
-        echo >&2 "Error: Installing old-style SPKGs is no longer supported"
-        exit 1
+        # Do a final check that PKG_SRC is a file with an absolute path
+        cd /
+        if [ ! -f "$PKG_SRC" ]; then
+            echo >&2 "Error: spkg file '$PKG_SRC' not found."
+            echo >&2 "This shouldn't happen, it is a bug in the sage-spkg script."
+            exit 1
+        fi
     fi
 fi
 
-# Do a final check that PKG_SRC is a file with an absolute path
-cd /
-if [ ! -f "$PKG_SRC" ]; then
-    echo >&2 "Error: spkg file '$PKG_SRC' not found."
-    echo >&2 "This shouldn't happen, it is a bug in the sage-spkg script."
-    exit 1
-fi
-
 # Go back to SAGE_ROOT where we have less chance of completely messing
 # up the system if we do something wrong.
 cd "$SAGE_ROOT" || exit $?
 
 # If SAGE_SPKG_COPY_UPSTREAM is set, it should be the name of a directory
 # to which all upstream files are copied. This is used in sage-sdist.
-if [ -n "$SAGE_SPKG_COPY_UPSTREAM" ]; then
+if [ -n "$SAGE_SPKG_COPY_UPSTREAM" -a -n "$PKG_NAME_UPSTREAM" ]; then
     mkdir -p "$SAGE_SPKG_COPY_UPSTREAM" && cp -p "$PKG_SRC" "$SAGE_SPKG_COPY_UPSTREAM"
     if [ $? -ne 0 ]; then
         error_msg "Error copying upstream tarball to directory '$SAGE_SPKG_COPY_UPSTREAM'"
@@ -483,14 +483,26 @@ extract_the_package() { ##########################################
 
 cd "$SAGE_BUILD_DIR" || exit $?
 echo "Setting up build directory for $PKG_NAME"
-cp -RLp "$PKG_SCRIPTS" "$PKG_NAME"
-cd "$PKG_NAME" || exit $?
-
-if [ "$SAGE_KEEP_BUILT_SPKGS" = "yes" ]; then
-    touch .keep
-fi
-
-case "$PKG_SRC" in
+if [ -z "$PKG_NAME_UPSTREAM" ]; then
+    # Not a normal or wheel package.
+    # Transplant the 'src' symlink, copy scripts.
+    mkdir -p "$PKG_NAME"
+    if [ -d "$PKG_SCRIPTS"/src ]; then
+        ln -s $(cd "$PKG_SCRIPTS"/src && pwd -P) "$PKG_NAME"/src
+    fi
+    for a in build install check preinst postinst; do  # replace by use of $WRAPPED_SCRIPTS later
+        if [ -r "$PKG_SCRIPTS"/spkg-$a.in ]; then
+            cp "$PKG_SCRIPTS"/spkg-$a.in "$PKG_NAME"/
+        elif [ -x "$PKG_SCRIPTS"/spkg-$a ]; then
+            cp "$PKG_SCRIPTS"/spkg-$a "$PKG_NAME"/
+        fi
+    done
+    cd "$PKG_NAME" || exit $?
+else
+    # Normal or wheel package, copy whole directory, resolving symlinks
+    cp -RLp "$PKG_SCRIPTS" "$PKG_NAME"
+    cd "$PKG_NAME" || exit $?
+    case "$PKG_SRC" in
     *.whl)
         # (Platform-independent) wheel
         # Do not extract, do not create a src directory,
@@ -514,7 +526,11 @@ case "$PKG_SRC" in
         fi
         cd ..
         ;;
-esac
+    esac
+fi
+if [ "$SAGE_KEEP_BUILT_SPKGS" = "yes" ]; then
+    touch .keep
+fi
 } ############################################ extract_the_package
 
 # The package has been extracted,
@@ -619,10 +635,14 @@ $CC -v
 echo "****************************************************"
 
 # Poison the proxy variable to forbid downloads in spkg-install
-export http_proxy=http://192.0.2.0:5187/
-export https_proxy=$http_proxy
-export ftp_proxy=$http_proxy
-export rsync_proxy=$http_proxy
+# for normal packages
+if [ -n "$PKG_NAME_UPSTREAM" ]; then
+    # Normal/wheel package
+    export http_proxy=http://192.0.2.0:5187/
+    export https_proxy=$http_proxy
+    export ftp_proxy=$http_proxy
+    export rsync_proxy=$http_proxy
+fi
 
 # We need to run sage-rebase.sh for each package installed, but it
 # can be dangerous to do this while other packages are installing

build/make/Makefile.in

@@ -725,7 +725,7 @@ $(1)-$(4)-no-deps:
 	    echo "Error: The installation tree $(4) has been disabled" 2>&1; \
 	    echo "$$($(4)_DISABLED_MESSAGE)" 2>&1; \
 	    exit 1; \
-	elif [ -x '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install' ]; then \
+	elif [ -x '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install' -o -r '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install.in' ]; then \
 	    rm -rf '$$($(4))/var/lib/sage/scripts/$(1)'; \
 	    cd '$$(SAGE_ROOT)/build/pkgs/$(1)' && \
 		. '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \
@@ -740,9 +740,9 @@ $(1)-$(4)-no-deps:
 		SAGE_SPKG_SCRIPTS=$$($(4))/var/lib/sage/scripts \
 		SAGE_INST_LOCAL=$$($(4)) \
 		SAGE_CHECK=$$(SAGE_CHECK_$(1)) \
-		sage-logger -p '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install && if [ $$$$SAGE_CHECK != no -a -x $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check ]; then $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check; fi' '$$(SAGE_LOGS)/$(1)-$(2).log' && \
-	    rm -f "$$($(4))/$(SPKG_INST_RELDIR)/$(1)"-* && \
-	    touch "$$($(4))/$(SPKG_INST_RELDIR)/$(1)-$(2)"; \
+		sage-logger -p 'SAGE_CHECK=$$(SAGE_CHECK_$(1)) PATH=$$(SAGE_SRC)/bin:$$($(4))/bin:$$$$PATH $$(SAGE_SPKG) $$(SAGE_SPKG_OPTIONS) \
+		$(if $(filter $(1),$(TOOLCHAIN_DEPS)),--keep-existing) \
+		$(1)-$(2) $$($(4))' '$$(SAGE_LOGS)/$(1)-$(2).log'; \
 	else ( \
 	    echo; \
 	    echo "Note: $(1) is a dummy package that the Sage distribution uses"; \
@@ -781,14 +781,10 @@ $(1)-$(4)-check:
 $(1)-check: $(1)-$(4)-check
 
 $(1)-$(4)-uninstall:
-	-$(AM_V_at)cd '$$(SAGE_ROOT)/build/pkgs/$(1)' && \
-		. '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \
-		. '$$(SAGE_ROOT)/src/bin/sage-env-config' && \
-		. '$$(SAGE_ROOT)/src/bin/sage-env' && \
-		. '$$(SAGE_ROOT)/build/bin/sage-build-env-config' && \
-		. '$$(SAGE_ROOT)/build/bin/sage-build-env' && \
-		'$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-uninstall'
-	-rm -f "$$($(4))/$(SPKG_INST_RELDIR)/$(1)"-*
+	if [ -d '$$($(4))' ]; then \
+	    sage-spkg-uninstall $(if $(filter $(1),$(TOOLCHAIN_DEPS)),--keep-files) \
+		$(1) '$$($(4))'; \
+	fi
 
 $(1)-uninstall: $(1)-$(4)-uninstall
 

build/pkgs/sage_conf/spkg-install.in

@@ -0,0 +1,9 @@
+cd src
+if [ "$SAGE_EDITABLE" = yes ]; then
+    sdh_pip_editable_install .
+    if [ "$SAGE_WHEELS" = yes ]; then
+        sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
+else
+    sdh_pip_install .
+fi

build/pkgs/sage_docbuild/spkg-install.in

@@ -0,0 +1,9 @@
+cd src
+if [ "$SAGE_EDITABLE" = yes ]; then
+    sdh_pip_editable_install .
+    if [ "$SAGE_WHEELS" = yes ]; then
+        sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
+else
+    sdh_pip_install .
+fi

build/pkgs/sage_setup/spkg-install.in

@@ -0,0 +1,9 @@
+cd src
+if [ "$SAGE_EDITABLE" = yes ]; then
+    sdh_pip_editable_install .
+    if [ "$SAGE_WHEELS" = yes ]; then
+        sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
+else
+    sdh_pip_install .
+fi

build/pkgs/sage_sws2rst/spkg-install.in

@@ -0,0 +1,9 @@
+cd src
+if [ "$SAGE_EDITABLE" = yes ]; then
+    sdh_pip_editable_install .
+    if [ "$SAGE_WHEELS" = yes ]; then
+        sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
+else
+    sdh_pip_install .
+fi

build/pkgs/sagelib/spkg-install → build/pkgs/sagelib/spkg-install.in

@@ -1,15 +1,3 @@
-#!/usr/bin/env bash
-# From sage-spkg.
-# For type=script packages, the build rule in build/make/Makefile sources
-# sage-env but not sage-dist-helpers.
-lib="$SAGE_ROOT/build/bin/sage-dist-helpers"
-source "$lib"
-if [ $? -ne 0 ]; then
-    echo >&2 "Error: failed to source $lib"
-    echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?"
-    exit 1
-fi
-
 if [ "$SAGE_EDITABLE" = yes ]; then
     cd "$SAGE_SRC"
 else

build/pkgs/sagemath_bliss/spkg-install → build/pkgs/sagemath_bliss/spkg-install.in

@@ -1,14 +1,3 @@
-#!/usr/bin/env bash
-# From sage-spkg.
-# For type=script packages, the build rule in build/make/Makefile sources
-# sage-env but not sage-dist-helpers.
-lib="$SAGE_ROOT/build/bin/sage-dist-helpers"
-source "$lib"
-if [ $? -ne 0 ]; then
-    echo >&2 "Error: failed to source $lib"
-    echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?"
-    exit 1
-fi
 cd src
 
 export PIP_NO_INDEX=true

build/pkgs/sagemath_categories/spkg-install.in

@@ -0,0 +1 @@
+../sagemath_objects/spkg-install.in

build/pkgs/sagemath_coxeter3/spkg-install → build/pkgs/sagemath_coxeter3/spkg-install.in

@@ -1,14 +1,3 @@
-#!/usr/bin/env bash
-# From sage-spkg.
-# For type=script packages, the build rule in build/make/Makefile sources
-# sage-env but not sage-dist-helpers.
-lib="$SAGE_ROOT/build/bin/sage-dist-helpers"
-source "$lib"
-if [ $? -ne 0 ]; then
-    echo >&2 "Error: failed to source $lib"
-    echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?"
-    exit 1
-fi
 cd src
 
 export PIP_NO_INDEX=true

build/pkgs/sagemath_environment/spkg-install.in

@@ -0,0 +1 @@
+../sagemath_objects/spkg-install.in