Trac #27824: spkg-configure.m4 for python3

Do not install python3 if a sufficiently new version is available in
`$PATH`. This could be a system python3, or the python3 from within a
venv.

As suggested in #29032, we make $SAGE_LOCAL a venv
(https://docs.python.org/3/library/venv.html) over the system python3 --
if a suitable system python3 is found -- by using the `venv.EnvBuilder`
API.

This is done by the new script `build/bin/sage-venv`, which we use both
during `configure` (to make sure that we select a system python3 for
which venvs work correctly) and during `make`.

The venv does not include the system site-packages: We continue to
install all Python packages into our venv using the existing build
infrastructure. We keep the task of using system site-packages for the
follow-up ticket #29023.

(Note, we use `venv` (new since Python 3.3), not `virtualenv`. So this
change is limited to Python 3 builds of Sage.)

URL: https://trac.sagemath.org/27824
Reported by: dimpase
Ticket author(s): Matthias Koeppe, Erik Bray
Reviewer(s): Dima Pasechnik

.gitignore

@@ -13,6 +13,7 @@
 /config.log
 /config.status
 /configure
+/conftest*
 
 /m4/sage_spkg_configures.m4
 

build/bin/sage-venv

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+# (Actually, we invoke this script with a specific python3 determined by configure.)
+
+# Adapted from examples in https://docs.python.org/3/library/venv.html
+
+import venv
+import os
+import argparse
+
+parser = argparse.ArgumentParser(prog=__name__,
+                                 description='Creates a virtual Python '
+                                             'environment in a target directory.')
+parser.add_argument('env_dir', metavar='ENV_DIR',
+                    help='A directory in which to create the'
+                         'virtual environment.')
+
+parser.add_argument('--system-site-packages', default=False,
+                    action='store_true', dest='system_site',
+                    help='Give the virtual environment access to the '
+                         'system site-packages dir.')
+
+parser.add_argument('--clear', default=False, action='store_true',
+                    dest='clear', help='Delete the contents of the '
+                                       'virtual environment '
+                                       'directory if it already '
+                                       'exists, before virtual '
+                                       'environment creation.')
+parser.add_argument('--upgrade', default=False, action='store_true',
+                    dest='upgrade', help='Upgrade the virtual '
+                                         'environment directory to '
+                                         'use this version of '
+                                         'Python, assuming Python '
+                                         'has been upgraded '
+                                         'in-place.')
+
+options = parser.parse_args()
+if options.upgrade and options.clear:
+    raise ValueError('you cannot supply --upgrade and --clear together.')
+
+if os.name == 'nt':
+    # default for Windows
+    use_symlinks = False
+else:
+    # default for posix
+    # On macOS, definitely need symlinks=True (which matches what we test in build/pkgs/spkg-configure.m4)
+    # or it may fail with 'dyld: Library not loaded: @executable_path/../Python3' on macOS.
+    use_symlinks = True
+
+b = venv.EnvBuilder(system_site_packages=options.system_site,
+                    clear=options.clear,
+                    upgrade=options.upgrade,
+                    symlinks=use_symlinks)
+c = b.ensure_directories(options.env_dir)
+b.setup_python(c)
+b.create_configuration(c)
+# We do not call setup_scripts, which would install the venv 'activate'/'deactivate' scripts.
+

build/make/deps

@@ -29,7 +29,24 @@ STARTED = $(SAGE_LOCAL)/etc/sage-started.txt
 	sagelib \
 	doc doc-html doc-html-jsmath doc-html-mathjax doc-pdf \
 	doc-clean doc-src-clean doc-output-clean \
-	clean sagelib-clean build-clean _clean-broken-gcc
+	clean sagelib-clean build-clean python3_venv _clean-broken-gcc
+
+ifneq ($(PYTHON_FOR_VENV),)
+# Special rule for making the Python virtualenv from the system Python (Python
+# 3 only).  $(PYTHON) is set in Makefile to python3_venv.
+# Thus $(inst_python3_venv) will be the dependency of every Python package.
+#
+# TODO: If we reconfigure to build our own Python after having used the system
+# Python, files installed to create the virtualenv should be *removed*.  That
+# could either be done here by the makefile, or in an spkg-preinst for python3
+ifeq ($(PYTHON),python3)
+PYTHON = python3_venv
+endif
+inst_python3_venv = $(SAGE_LOCAL)/pyvenv.cfg
+
+$(inst_python3_venv):
+	$(PYTHON_FOR_VENV) $(SAGE_ROOT)/build/bin/sage-venv "$(SAGE_LOCAL)"
+endif
 
 # Build everything and start Sage.
 # Note that we put the "doc" target first in the rule below because

build/pkgs/python3/spkg-configure.m4

@@ -0,0 +1,96 @@
+SAGE_SPKG_CONFIGURE([python3], [
+   SAGE_SPKG_DEPCHECK([sqlite libpng bzip2 xz libffi], [
+      AS_IF([test $SAGE_PYTHON_VERSION = 2], [
+        dnl If we use Python 2 for Sage, we install Python 3 too and do NOT attempt to do
+        dnl venv using system python3 over SAGE_LOCAL.
+        dnl (In particular, the setuptools and pip install scripts are not prepared for
+        dnl handling this situation.)
+        sage_spkg_install_python3=yes
+      ], [
+        dnl Using Python 3 for Sage.  Check if we can do venv with a system python3
+        dnl instead of building our own copy.
+        check_modules="sqlite3, ctypes, math, hashlib, crypt, readline, socket, zlib, distutils.core"
+        AC_CACHE_CHECK([for python3 >= 3.7.3, < 3.8 with modules $check_modules], [ac_cv_path_PYTHON3], [
+            AC_MSG_RESULT([])
+            AC_PATH_PROGS_FEATURE_CHECK([PYTHON3], [python3.7 python3], [
+                AC_MSG_CHECKING([... whether $ac_path_PYTHON3 is good])
+                python3_version=`"$ac_path_PYTHON3" --version 2>&1 \
+                    | $SED -n -e 's/\([[0-9]]*\.[[0-9]]*\.[[0-9]]*\).*/\1/p'`
+                AS_IF([test -n "$python3_version"], [
+                    AX_COMPARE_VERSION([$python3_version], [ge], [3.7.3], [
+                        AX_COMPARE_VERSION([$python3_version], [lt], [3.8.0], [
+                            dnl Because the system python is not used directly but rather in a venv without site-packages,
+                            dnl we test whether the module will be available in a venv.
+                            dnl Otherwise, some system site-package may be providing this module to the system python.
+                            dnl m4_define([conftest_venv], [config-venv]) .... for debugging only
+                            rm -rf conftest_venv
+                            AS_IF(["$ac_path_PYTHON3" build/bin/sage-venv conftest_venv && conftest_venv/bin/python3 -c "import $check_modules"], [
+                                AC_LANG_PUSH([C])
+                                AC_LANG_CONFTEST([
+                                    AC_LANG_SOURCE([[
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+static PyMethodDef SpamMethods[] = {
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+static struct PyModuleDef spammodule = {
+    PyModuleDef_HEAD_INIT,
+    "spam",   /* name of module */
+    NULL,     /* module documentation, may be NULL */
+    -1,       /* size of per-interpreter state of the module,
+                 or -1 if the module keeps state in global variables. */
+    SpamMethods
+};
+PyMODINIT_FUNC
+PyInit_spam(void)
+{
+    PyObject *m;
+
+    m = PyModule_Create(&spammodule);
+    return m;
+}
+                                    ]])
+                                ])
+                                AC_LANG_POP([C])
+                                cat > conftest.py <<EOF
+from distutils.core import setup
+from distutils.extension import Extension
+from sys import exit
+modules = list((Extension("config_check_distutils", list(("conftest.c",))),))
+setup(name="config_check_distutils", ext_modules=modules)
+exit(0)
+EOF
+                                AS_IF([conftest_venv/bin/python3 conftest.py --quiet build --build-base=conftest.dir], [
+                                    ac_cv_path_PYTHON3="$ac_path_PYTHON3"
+                                    ac_path_PYTHON3_found=:
+                                    AC_MSG_RESULT([yes])
+                                    dnl introduction for AC_MSG_RESULT printed by AC_CACHE_CHECK
+                                    AC_MSG_CHECKING([for python3 >= 3.7.3, < 3.8 with modules $check_modules])
+                                ], [
+                                    AC_MSG_RESULT([no, the version is in the supported range, and the modules can be imported, but distutils cannot build an extension])
+                                ])
+                            ], [
+                                AC_MSG_RESULT([no, the version is in the supported range but cannot import one of the required modules: $check_modules])
+                            ])
+                        ], [
+                            AC_MSG_RESULT([no, $python3_version is too recent])
+                        ])
+                    ], [
+                        AC_MSG_RESULT([no, $python3_version is too old])
+                    ])
+                ], [
+                    AC_MSG_RESULT([no, "$ac_path_PYTHON3 --version" does not work])
+                ])
+            ])
+        ])
+        AS_IF([test -z "$ac_cv_path_PYTHON3"],
+              [sage_spkg_install_python3=yes])
+      ])
+   ])
+],, [
+    dnl PRE
+], [
+    dnl POST
+    AS_IF([test x$sage_spkg_install_python3 = xno], [PYTHON_FOR_VENV="$ac_cv_path_PYTHON3"])
+    AC_SUBST([PYTHON_FOR_VENV])
+])

src/bin/sage-env

@@ -312,6 +312,8 @@ export UNAME=`uname | sed 's/CYGWIN.*/CYGWIN/' `
 
 # Mac OS X-specific setup
 if [ "$UNAME" = "Darwin" ]; then
+  # Avoid 'MACOSX_DEPLOYMENT_TARGET mismatch: now "10.9" but "10.15" during configure'
+  if [ -z "$PYTHON_FOR_VENV" ]; then
     # set MACOSX_DEPLOYMENT_TARGET -- if set incorrectly, this can
     # cause lots of random "Abort trap" issues on OSX. see trac
     # #7095 for an example.
@@ -324,7 +326,7 @@ if [ "$UNAME" = "Darwin" ]; then
         MACOSX_DEPLOYMENT_TARGET=10.$[$MACOSX_VERSION-4]
     fi
     export MACOSX_DEPLOYMENT_TARGET MACOSX_VERSION
-
+  fi
     # Work around problems on recent OS X crashing with an error message
     # "... may have been in progress in another thread when fork() was called"
     # when objective-C functions are called after fork(). See Trac #25921.

src/bin/sage-env-config.in

@@ -36,6 +36,8 @@ if [ "$SAGE_PYTHON_VERSION" = 3 ]; then
     export SAGE_PYTHON3=yes
 fi
 
+export PYTHON_FOR_VENV="@PYTHON_FOR_VENV@"
+
 export SAGE_PKG_CONFIG_PATH="@SAGE_PKG_CONFIG_PATH@"
 if [ -n "$SAGE_PKG_CONFIG_PATH" ]; then
     # set up external pkg-config to look into SAGE_LOCAL/lib/pkgconfig/

src/sage/env.py

@@ -205,9 +205,9 @@ def var(key, *fallbacks, **kwds):
 
 def _get_shared_lib_filename(libname, *additional_libnames):
     """
-    Return the full path to a shared library file installed in the standard
-    location for the system within the ``LIBDIR`` prefix (or
-    ``$SAGE_LOCAL/lib`` in the case of manual build of Sage).
+    Return the full path to a shared library file installed in
+    ``$SAGE_LOCAL/lib`` or the directories associated with the
+    Python sysconfig.
 
     This can also be passed more than one library name (e.g. for cases where
     some library may have multiple names depending on the platform) in which
@@ -244,11 +244,17 @@ def _get_shared_lib_filename(libname, *additional_libnames):
 
     for libname in (libname,) + additional_libnames:
         if sys.platform == 'cygwin':
-            bindir = sysconfig.get_config_var('BINDIR')
+            # Later down we take the last matching DLL found, so search
+            # SAGE_LOCAL second so that it takes precedence
+            bindirs = [
+                sysconfig.get_config_var('BINDIR'),
+                os.path.join(SAGE_LOCAL, 'bin')
+            ]
             pats = ['cyg{}.dll'.format(libname), 'cyg{}-*.dll'.format(libname)]
             filenames = []
-            for pat in pats:
-                filenames += glob.glob(os.path.join(bindir, pat))
+            for bindir in bindirs:
+                for pat in pats:
+                    filenames += glob.glob(os.path.join(bindir, pat))
 
             # Note: This is not very robust, since if there are multi DLL
             # versions for the same library this just selects one more or less
@@ -262,10 +268,13 @@ def _get_shared_lib_filename(libname, *additional_libnames):
             else:
                 ext = 'so'
 
-            libdirs = [sysconfig.get_config_var('LIBDIR')]
+            libdirs = [
+                os.path.join(SAGE_LOCAL, 'lib'),
+                sysconfig.get_config_var('LIBDIR')
+            ]
             multilib = sysconfig.get_config_var('MULTILIB')
             if multilib:
-                libdirs.insert(0, os.path.join(libdirs[0], multilib))
+                libdirs.insert(1, os.path.join(libdirs[0], multilib))
 
             for libdir in libdirs:
                 basename = 'lib{}.{}'.format(libname, ext)