Add support for buildPython version 3.12 (closes #931)

.github/workflows/ci.yml

@@ -161,11 +161,13 @@ jobs:
       - uses: ./.github/actions/setup-python
         id: setup-python
         with:
-          # This should match OLD_BUILD_PYTHON_VERSION and MIN_BUILD_PYTHON_VERSION
-          # in test_gradle_plugin.
+          # This should include OLD_BUILD_PYTHON_VERSION, MIN_BUILD_PYTHON_VERSION and
+          # MAX_BUILD_PYTHON_VERSION from test_gradle_plugin, if they're not already
+          # returned by `list-versions.py --minor`.
           extra-versions: |
             3.6
             3.7
+            3.12
 
       - name: Download Maven repository
         uses: actions/download-artifact@v3.0.2

product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java

@@ -22,6 +22,7 @@ public class Common {
         PYTHON_VERSIONS.put("3.9.13", "1");
         PYTHON_VERSIONS.put("3.10.6", "1");
         PYTHON_VERSIONS.put("3.11.0", "2");
+        // TODO: once we add 3.12, remove it from extra-versions in ci.yml
     }
 
     public static List<String> PYTHON_VERSIONS_SHORT = new ArrayList<>();
@@ -81,6 +82,17 @@ public static String assetZip(String type, String abi) {
     public static final String ASSET_BUILD_JSON = "build.json";
     public static final String ASSET_CACERT = "cacert.pem";
 
+    public static String osName() {
+        String property = System.getProperty("os.name");
+        String[] knownNames = new String[] {"linux", "mac", "windows"};
+        for (String name : knownNames) {
+            if (property.toLowerCase(Locale.ENGLISH).startsWith(name)) {
+                return name;
+            }
+        }
+        throw new RuntimeException("unknown os.name: " + property);
+    }
+
     public static String findExecutable(String name) throws FileNotFoundException {
         File file = new File(name);
         if (file.isAbsolute()) {

product/gradle-plugin/README.md

@@ -103,8 +103,8 @@ do the following:
 
 Check out the upstream-pip branch.
 
-Delete the relevant directories in src/main/python, including the .dist-info directory. Note
-that pkg_resources is part of setuptools.
+Delete the package from src/main/python, including the .dist-info directory. Note that
+setuptools includes some files outside of its main directory.
 
 Download the wheel of the new version, and unpack it into src/main/python.
 

product/gradle-plugin/build.gradle.kts

@@ -1,6 +1,7 @@
 import com.chaquo.python.internal.BuildCommon
 import com.chaquo.python.internal.Common
 import com.chaquo.python.internal.Common.findExecutable
+import com.chaquo.python.internal.Common.osName
 
 plugins {
     `java-gradle-plugin`
@@ -83,7 +84,7 @@ abstract class TestPythonTask : DefaultTask() {
         pb.directory(File(workingDir))
         pb.environment().putAll(environment)
 
-        command += if (System.getProperty("os.name").toLowerCase().contains("windows")) {
+        command += if (osName() == "windows") {
             listOf(findExecutable("py"), "-$pythonVersion")
         } else {
             listOf(findExecutable("python$pythonVersion"))

product/gradle-plugin/src/main/kotlin/PythonPlugin.kt

@@ -135,12 +135,12 @@ class PythonPlugin : Plugin<Project> {
             "python", extension.defaultConfig)
 
         android.productFlavors.all {
-            val python = extension.productFlavors.maybeCreate(name)
+            val python = extension.productFlavors.maybeCreate(name)  // New DSL
             (this as ExtensionAware).extensions.add("python", python)  // Old DSL
         }
 
         android.sourceSets.all {
-           val dirSet = extension.sourceSets.maybeCreate(name)
+           val dirSet = extension.sourceSets.maybeCreate(name)  // New DSL
            (this as ExtensionAware).extensions.add("python", dirSet)  // Old DSL
         }
    }

product/gradle-plugin/src/main/kotlin/PythonTasks.kt

@@ -3,6 +3,7 @@ package com.chaquo.python
 import com.android.build.api.variant.*
 import com.chaquo.python.internal.*
 import com.chaquo.python.internal.Common.assetZip
+import com.chaquo.python.internal.Common.osName
 import org.apache.commons.compress.archivers.zip.*
 import org.gradle.api.*
 import org.gradle.api.artifacts.*
@@ -24,9 +25,7 @@ internal class TaskBuilder(
     val abis: List<String>
 ) {
     val project = plugin.project
-    lateinit var buildPython: List<String>
-
-    lateinit var buildPackagesTask: Provider<OutputDirTask>
+    lateinit var buildPackagesTask: Provider<BuildPackagesTask>
     lateinit var srcTask: Provider<OutputDirTask>
     lateinit var reqsTask: Provider<OutputDirTask>
 
@@ -56,33 +55,82 @@ internal class TaskBuilder(
         }
     }
 
-    fun createBuildPackagesTask(): Provider<OutputDirTask> {
+    fun createBuildPackagesTask(): Provider<BuildPackagesTask> {
         val taskName = "extractPythonBuildPackages"
         try {
-            return project.tasks.named<OutputDirTask>(taskName)
+            return project.tasks.named<BuildPackagesTask>(taskName)
         } catch (e: UnknownDomainObjectException) {
-            return project.tasks.register<OutputDirTask>(taskName) {
+            return project.tasks.register<BuildPackagesTask>(taskName) {
+                var bp: List<String>?
+                try {
+                    bp = findBuildPython()
+                } catch (e: BuildPythonException) {
+                    bp = null
+                    exception = e
+                }
+                inputs.property("buildPython", bp).optional(true)
+
                 // Keep the path short to avoid the the Windows 260-character limit.
-                outputFiles = project.fileTree(plugin.buildSubdir("bp")) {
+                outputFiles = project.fileTree(plugin.buildSubdir("env")) {
                     exclude("**/__pycache__")
                 }
-                doLast {
-                    val zipPath = plugin.extractResource(
-                        "gradle/build-packages.zip", plugin.buildSubdir())
-                    project.copy {
-                        from(project.zipTree(zipPath))
-                        into(outputDir)
+
+                if (bp != null) {
+                    doLast {
+                        project.exec {
+                            commandLine(bp)
+                            args("-m", "venv", "--without-pip", outputDir)
+                        }
+
+                        val zipPath = plugin.extractResource(
+                            "gradle/build-packages.zip", plugin.buildSubdir())
+                        project.copy {
+                            from(project.zipTree(zipPath))
+                            into(sitePackages)
+                        }
+                        project.delete(zipPath)
                     }
-                    project.delete(zipPath)
                 }
             }
         }
     }
 
+    open class BuildPackagesTask : OutputDirTask() {
+        @get:Internal
+        lateinit var exception: Exception
+
+        @get:Internal
+        val pythonExecutable by lazy {
+            if (::exception.isInitialized) {
+                throw exception
+            } else if (osName() == "windows") {
+                outputDir.resolve("Scripts/python.exe")
+            } else {
+                outputDir.resolve("bin/python")
+            }
+        }
+
+        @get:Internal
+        val sitePackages by lazy {
+            if (osName() == "windows") {
+                outputDir.resolve("Lib/site-packages")
+            } else {
+                val libDir = outputDir.resolve("lib")
+                val pythonDirs = libDir.listFiles()!!.filter {
+                    it.name.startsWith("python")
+                }
+                if (pythonDirs.size != 1) {
+                    throw GradleException(
+                        "found ${pythonDirs.size} python directories in $libDir")
+                }
+                pythonDirs[0].resolve("site-packages")
+            }
+        }
+    }
+
     fun createSrcTask() =
         registerTask("merge", "sources") {
             inputs.files(buildPackagesTask)
-            inputs.property("buildPython", python.buildPython).optional(true)
             inputs.property("pyc", python.pyc.src).optional(true)
 
             val dirSets = ArrayList<SourceDirectorySet>()
@@ -149,7 +197,6 @@ internal class TaskBuilder(
             inputs.files(buildPackagesTask)
             inputs.property("abis", abis)
             inputs.property("minApiLevel", variant.minSdkVersion.apiLevel)
-            inputs.property("buildPython", python.buildPython).optional(true)
             inputs.property("pip", python.pip)
             inputs.property("pyc", python.pyc.pip).optional(true)
 
@@ -261,7 +308,6 @@ internal class TaskBuilder(
         val outputDir = plugin.buildSubdir("proxies", variant)
         val task = registerTask("generate", "proxies") {
             inputs.files(buildPackagesTask, reqsTask, srcTask)
-            inputs.property("buildPython", python.buildPython).optional(true)
             inputs.property("staticProxy", python.staticProxy)
 
             this.outputDir = outputDir
@@ -542,16 +588,9 @@ internal class TaskBuilder(
     }
 
     fun execBuildPython(configure: ExecSpec.() -> Unit) {
-        if (! ::buildPython.isInitialized) {
-            buildPython = findBuildPython()
-        }
-
         try {
             project.exec {
-                environment("PYTHONPATH", buildPackagesTask.get().outputDir)
-                commandLine(buildPython)
-                args("-S")  // Avoid interference from site-packages. This is not inherited by
-                            // subprocesses, so it's used again in pip_install.py.
+                executable(buildPackagesTask.get().pythonExecutable)
                 configure()
             }
         } catch (e: ExecException) {
@@ -579,7 +618,7 @@ internal class TaskBuilder(
             } else {
                 val version = python.version!!
                 for (suffix in listOf(version, version.split(".")[0])) {
-                    if (System.getProperty("os.name").startsWith("Windows")) {
+                    if (osName() == "windows") {
                         // See PEP 397. After running the official Windows installer
                         // with default settings, this will be the only Python thing on
                         // the PATH.

product/gradle-plugin/src/main/python/chaquopy/pip_install.py

@@ -111,9 +111,6 @@ def pip_install(self, abi, reqs):
             # Warning: `pip install --target` is very simple-minded: see
             # https://github.com/pypa/pip/issues/4625#issuecomment-375977073.
             cmdline = ([sys.executable,
-                       "-S",  # Avoid interference from site-packages. This is not inherited
-                              # by subprocesses, so it's used again in pip (see wheel.py and
-                              # req_install.py).
                         "-m", "pip", "install",
                         "--isolated",  # Disables environment variables.
                         "--target", abi_dir,

product/gradle-plugin/src/main/python/chaquopy_monkey.py

@@ -1,3 +1,8 @@
+import os
+import sys
+import types
+
+
 # We want to cause a quick and comprehensible failure when a package attempts to build
 # native code, while still allowing a pure-Python fallback if available. This is tricky,
 # because different packages have different approaches to pure-Python fallbacks:
@@ -30,18 +35,27 @@
 # list (e.g. minorminer, lz4), but the error messages from these packages aren't too bad, and
 # I've never seen one which has a pure-Python fallback.
 def disable_native():
+    disable_native_distutils()
+    disable_native_environ()
+
+
+def disable_native_distutils():
     # Recent versions of setuptools redirect distutils to their own bundled copy, so try
     # to import that first.
     try:
         import setuptools  # noqa: F401
     except ImportError:
         pass
 
+    try:
+        import distutils  # noqa: F401
+    except ImportError:
+        # distutils was removed in Python 3.12, so it will only exist if setuptools is
+        # in the build environment.
+        return
+
     from distutils import ccompiler
     from distutils.unixccompiler import UnixCCompiler
-    import os
-    import sys
-    import types
 
     ccompiler.get_default_compiler = lambda *args, **kwargs: "disabled"
     ccompiler.compiler_class["disabled"] = (
@@ -73,6 +87,8 @@ def link(*args, **kwargs):
     disabled_mod.DisabledCompiler = DisabledCompiler
     sys.modules[disabled_mod_name] = disabled_mod
 
+
+def disable_native_environ():
     # Try to disable native builds for packages which don't use the distutils native build
     # system at all (e.g. uwsgi), or only use it to wrap an external build script (e.g. pynacl).
     for tool in ["ar", "as", "cc", "cxx", "ld"]:

product/gradle-plugin/src/main/python/pip/_internal/utils/setuptools_build.py

@@ -41,12 +41,6 @@ def make_setuptools_shim_args(
     if unbuffered_output:
         args += ["-u"]
 
-    # Chaquopy: added '-S' to avoid interference from site-packages. This makes
-    # non-installable packages fail more quickly and consistently. Also, some packages
-    # (e.g. Cython) install distutils hooks which can interfere with our attempts to
-    # disable compilers in chaquopy_monkey.
-    args.append('-S')
-
     from pip._vendor.packaging import markers
     chaquopy_monkey = (
         "import chaquopy_monkey; chaquopy_monkey.disable_native()"

product/gradle-plugin/src/main/python/pip/_vendor/pkg_resources/__init__.py

@@ -2161,7 +2161,8 @@ def resolve_egg_link(path):
     return next(dist_groups, ())
 
 
-register_finder(pkgutil.ImpImporter, find_on_path)
+# Chaquopy: ImpImporter was removed in Python 3.12.
+# register_finder(pkgutil.ImpImporter, find_on_path)
 
 if hasattr(importlib_machinery, 'FileFinder'):
     register_finder(importlib_machinery.FileFinder, find_on_path)
@@ -2312,7 +2313,8 @@ def file_ns_handler(importer, path_item, packageName, module):
         return subpath
 
 
-register_namespace_handler(pkgutil.ImpImporter, file_ns_handler)
+# Chaquopy: ImpImporter was removed in Python 3.12.
+# register_namespace_handler(pkgutil.ImpImporter, file_ns_handler)
 register_namespace_handler(zipimport.zipimporter, file_ns_handler)
 
 if hasattr(importlib_machinery, 'FileFinder'):