Add spkg-configure.m4 for Python 3 to detect and use a system Python 3.

If a working system Python 3 is detected, the environment variable
SAGE_SYSTEM_PYTHON is set to the path of the Python interpreter
executable in sage-env-config.

Furthermore, if a system Python 3 is used, Sage Makefile will convert
SAGE_LOCAL into a virtualenv by using the Python 3 venv package--it
installs copies (should we change this to symlinks?) of the system
Python, and a pyvenv.cfg file to indicate to Python that SAGE_LOCAL is
an alternate virtualenv.  We use the venv module to do this rather than
the venv CLI, because this does not install the venv activation
scripts--we don't need these because sage-env already serves the same
purpose.

build/make/deps

@@ -29,7 +29,31 @@ STARTED = $(SAGE_LOCAL)/etc/sage-started.txt
 	sagelib \
 	doc doc-html doc-html-jsmath doc-html-mathjax doc-pdf \
 	doc-clean doc-src-clean doc-output-clean \
-	clean sagelib-clean build-clean _clean-broken-gcc
+	clean sagelib-clean build-clean python3_venv _clean-broken-gcc
+
+
+define SAGE_make_venv
+import venv
+b = venv.EnvBuilder()
+c = b.ensure_directories("$(SAGE_LOCAL)")
+b.setup_python(c)
+b.create_configuration(c)
+endef
+
+
+ifdef SAGE_SYSTEM_PYTHON
+# Special rule for making the Python virtualenv from the system Python (Python
+# 3 only)
+# TODO: If we reconfigure to build our own Python after having used the system
+# Python, files installed to create the virtualenv should be *removed*.  That
+# could either be done here by the makefile, or in an spkg-preinst for python3
+PYTHON = python3_venv
+inst_python3_venv = $(SAGE_LOCAL)/pyvenv.cfg
+
+export SAGE_make_venv
+$(inst_python3_venv):
+	$(SAGE_SYSTEM_PYTHON) -c "$$SAGE_make_venv"
+endif
 
 # Build everything and start Sage.
 # Note that we put the "doc" target first in the rule below because

build/pkgs/python3/spkg-build

@@ -107,7 +107,7 @@ fi
 
 echo "Testing importing of various modules..."
 import_errors=false
-test_modules="ctypes math hashlib crypt readline socket"
+test_modules="ctypes math hashlib crypt readline socket zlib"
 if [ "$UNAME" = "Darwin" ]; then
     test_modules="$test_modules _scproxy"
 fi

build/pkgs/python3/spkg-configure.m4

@@ -0,0 +1,78 @@
+SAGE_SPKG_CONFIGURE([python3], [
+    SAGE_SPKG_DEPCHECK([sqlite libpng bzip2 xz libffi], [
+        check_modules="sqlite3, ctypes, math, hashlib, crypt, readline, socket, zlib, distutils.core"
+        AC_CACHE_CHECK([for python3 >= 3.7, < 3.8 with sqlite3 module], [ac_cv_path_PYTHON3], [
+            AC_MSG_RESULT([])
+            AC_PATH_PROGS_FEATURE_CHECK([PYTHON3], [python3.7 python3], [
+                AC_MSG_CHECKING([... whether $ac_path_PYTHON3 is good])
+                python3_version=`"$ac_path_PYTHON3" --version 2>&1 \
+                    | $SED -n -e 's/\([[0-9]]*\.[[0-9]]*\.[[0-9]]*\).*/\1/p'`
+                AS_IF([test -n "$python3_version"], [
+                    AX_COMPARE_VERSION([$python3_version], [ge], [3.7.0], [
+                        AX_COMPARE_VERSION([$python3_version], [lt], [3.8.0], [
+                            AS_IF(["$ac_path_PYTHON3" -S -c "import $check_modules"], [
+                                AC_LANG_PUSH([C])
+                                AC_LANG_CONFTEST([
+                                    AC_LANG_SOURCE([[
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+static PyMethodDef SpamMethods[] = {
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+static struct PyModuleDef spammodule = {
+    PyModuleDef_HEAD_INIT,
+    "spam",   /* name of module */
+    NULL,     /* module documentation, may be NULL */
+    -1,       /* size of per-interpreter state of the module,
+                 or -1 if the module keeps state in global variables. */
+    SpamMethods
+};
+PyMODINIT_FUNC
+PyInit_spam(void)
+{
+    PyObject *m;
+
+    m = PyModule_Create(&spammodule);
+    return m;
+}
+                                    ]])
+                                ])
+                                AC_LANG_POP([C])
+                                cat > conftest.py <<EOF
+from distutils.core import setup
+from distutils.extension import Extension
+from sys import exit
+modules = list((Extension("config_check_distutils", list(("conftest.c",))),))
+setup(name="config_check_distutils", ext_modules=modules)
+exit(0)
+EOF
+                                AS_IF([$ac_path_PYTHON3 -S conftest.py --quiet build --build-base=conftest.dir], [
+                                    ac_cv_path_PYTHON3="$ac_path_PYTHON3"
+                                    ac_path_PYTHON3_found=:
+                                    AC_MSG_RESULT([yes])
+                                    dnl introduction for AC_MSG_RESULT printed by AC_CACHE_CHECK
+                                    AC_MSG_CHECKING([for python3 >= 3.7, < 3.8 with modules $check_modules])
+                                ], [
+                                    AC_MSG_RESULT([no, the version is in the supported range, and the modules can be imported, but distutils cannot build an extension])
+                                ])
+			    ], [
+                                AC_MSG_RESULT([no, the version is in the supported range but cannot import one of the required modules: $check_modules])
+                            ])
+                        ], [
+                            AC_MSG_RESULT([no, $python3_version is too recent])
+                        ])
+                    ], [
+                        AC_MSG_RESULT([no, $python3_version is too old])
+                    ])
+                ], [
+                    AC_MSG_RESULT([no, "$ac_path_PYTHON3 --version" does not work])
+                ])
+            ])
+        ])
+        AS_IF([test -z "$ac_cv_path_PYTHON3"],
+              [sage_spkg_install_python3=yes])
+    ])
+],,,[
+    AS_IF([test x$sage_spkg_install_python3 = xno], [SAGE_SYSTEM_PYTHON="$ac_cv_path_PYTHON3"])
+    AC_SUBST(SAGE_SYSTEM_PYTHON)
+])

src/bin/sage-env

@@ -410,6 +410,13 @@ if [ -n "$PYTHONHOME" ]; then
     unset PYTHONHOME
 fi
 
+# If using the system python, treat $SAGE_LOCAL as though its a virtualenv
+# This environment variable is not used by Python directly, but it is used by
+# some third-party tools that work with Python virtualenvs
+if [ -n "$SAGE_SYSTEM_PYTHON" ]; then
+    export VIRTUAL_ENV="$SAGE_LOCAL"
+fi
+
 LDFLAGS="-L$SAGE_LOCAL/lib -Wl,-rpath,$SAGE_LOCAL/lib $LDFLAGS"
 export LDFLAGS
 

src/bin/sage-env-config.in

@@ -36,6 +36,8 @@ if [ "$SAGE_PYTHON_VERSION" = 3 ]; then
     export SAGE_PYTHON3=yes
 fi
 
+export SAGE_SYSTEM_PYTHON="@SAGE_SYSTEM_PYTHON@"
+
 # This is usually blank if the system GMP is used, or $SAGE_LOCAL otherwise
 export SAGE_GMP_PREFIX="@SAGE_GMP_PREFIX@"
 export SAGE_GMP_INCLUDE="@SAGE_GMP_INCLUDE@"

src/sage/env.py

@@ -195,9 +195,10 @@ def var(key, *fallbacks, **kwds):
 
 def _get_shared_lib_filename(libname, *additional_libnames):
     """
-    Return the full path to a shared library file installed in the standard
-    location for the system within the ``LIBDIR`` prefix (or
-    ``$SAGE_LOCAL/lib`` in the case of manual build of Sage).
+
+    Return the full path to a shared library file installed in
+    ``$SAGE_LOCAL/lib`` or the directories associated with the Python
+    sysconfig.
 
     This can also be passed more than one library name (e.g. for cases where
     some library may have multiple names depending on the platform) in which
@@ -234,11 +235,17 @@ def _get_shared_lib_filename(libname, *additional_libnames):
 
     for libname in (libname,) + additional_libnames:
         if sys.platform == 'cygwin':
-            bindir = sysconfig.get_config_var('BINDIR')
+            # Later down we take the last matching DLL found, so search
+            # SAGE_LOCAL second so that it takes precedence
+            bindirs = [
+                sysconfig.get_config_var('BINDIR'),
+                os.path.join(SAGE_LOCAL, 'bin')
+            ]
             pats = ['cyg{}.dll'.format(libname), 'cyg{}-*.dll'.format(libname)]
             filenames = []
-            for pat in pats:
-                filenames += glob.glob(os.path.join(bindir, pat))
+            for bindir in bindirs:
+                for pat in pats:
+                    filenames += glob.glob(os.path.join(bindir, pat))
 
             # Note: This is not very robust, since if there are multi DLL
             # versions for the same library this just selects one more or less
@@ -252,10 +259,13 @@ def _get_shared_lib_filename(libname, *additional_libnames):
             else:
                 ext = 'so'
 
-            libdirs = [sysconfig.get_config_var('LIBDIR')]
+            libdirs = [
+                os.path.join(SAGE_LOCAL, 'lib'),
+                sysconfig.get_config_var('LIBDIR')
+            ]
             multilib = sysconfig.get_config_var('MULTILIB')
             if multilib:
-                libdirs.insert(0, os.path.join(libdirs[0], multilib))
+                libdirs.insert(1, os.path.join(libdirs[0], multilib))
 
             for libdir in libdirs:
                 basename = 'lib{}.{}'.format(libname, ext)