[WIP] Use preferences.jl to set python executable

.github/workflows/aot.yml

@@ -16,10 +16,11 @@ jobs:
         os:
           - ubuntu-latest
         architecture: [x64]
-        python-version: ['3.x']
+        python-version: ['3.9']
         julia-version:
+          - '1.6'
           - '1'
-          - '~1.7.0-rc1'
+          - '~1.7'
           # - 'nightly'  # TODO: reenable
       fail-fast: false
     env:
@@ -46,7 +47,6 @@ jobs:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.architecture }}
           show-versioninfo: true
-
       - run: julia -e 'using Pkg; pkg"add PackageCompiler@v1"'
 
       - run: aot/compile.jl

.github/workflows/conda.yml

@@ -18,14 +18,14 @@ jobs:
           - macos-latest
           - windows-latest
         architecture: [x64]
-        julia-version: ['1']
+        julia-version: ['1.6', '1']
         include:
           - os: windows-latest
             architecture: x86
             julia-version: '1'
       fail-fast: false
     env:
-      PYTHON: ""
+      PYPREFERENCES_PYTHON: "conda"
     name: Test
       Julia ${{ matrix.julia-version }}
       Conda
@@ -41,7 +41,13 @@ jobs:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.architecture }}
           show-versioninfo: true
-      - uses: julia-actions/julia-buildpkg@v1
+      - name: Install PyPreferences
+        shell: julia --project=@. {0}
+        run: |
+          using Pkg;
+          pkg"dev ./PyPreferences.jl";
+          using PyPreferences;
+          PyPreferences.use_conda();
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1

.github/workflows/system.yml

@@ -20,7 +20,7 @@ jobs:
         architecture: [x64]
         python-version: ['3.x']
         julia-version:
-          - '1.0'
+          - '1.6'
           - '1'
           - 'nightly'
         include:
@@ -34,14 +34,6 @@ jobs:
             architecture: x64
             python-version: '3.9'
             julia-version: '1'
-          - os: ubuntu-latest
-            architecture: x64
-            python-version: '3.x'
-            julia-version: '1.5'
-          - os: ubuntu-latest
-            architecture: x64
-            python-version: '3.x'
-            julia-version: '1.4'
           # Test Python 2.7 only with a few combinations (TODO: drop 2.7).
           # Note that it does not work in macOS:
           - os: ubuntu-latest
@@ -80,14 +72,21 @@ jobs:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.architecture }}
           show-versioninfo: true
-      - uses: julia-actions/julia-buildpkg@v1
-        env:
-          PYTHON: python
-      - run: julia test/check_deps_version.jl ${{ matrix.python-version }}
+      - name: Set PYTHON Env variable
+        run: echo "PYTHON=python" >> $GITHUB_ENV
+        if: ${{ matrix.python-version == '2.7' }}
+      - name: Install PyPreferences
+        shell: julia --project=@. {0}
+        run: |
+          using Pkg;
+          pkg"dev ./PyPreferences.jl"
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:
           file: ./lcov.info
           flags: unittests
           name: codecov-umbrella
+      - name: tmate session if tests fail
+        if: failure()
+        uses: mxschmitt/action-tmate@v3

.gitignore

@@ -1,8 +1,13 @@
 Manifest.toml
+LocalPreferences.toml
 aot/Manifest.toml
 aot/Project.toml
 aot/_julia_path
 aot/sys.*
 deps/deps.jl
 deps/PYTHON
 deps/build.log
+
+PyPreferences.jl/Manifest.toml
+
+.envrc

Project.toml

@@ -1,25 +1,27 @@
 name = "PyCall"
 uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 authors = ["Steven G. Johnson <stevenj@mit.edu>", "Yichao Yu <yyc1992@gmail.com>", "Takafumi Arakaki <aka.tkf@gmail.com>", "Simon Kornblith <simon@simonster.com>", "Páll Haraldsson <Pall.Haraldsson@gmail.com>", "Jon Malmaud <malmaud@gmail.com>", "Jake Bolewski <jakebolewski@gmail.com>", "Keno Fischer <keno@alumni.harvard.edu>", "Joel Mason <jobba1@hotmail.com>", "Jameson Nash <vtjnash@gmail.com>", "The JuliaPy development team"]
-version = "1.93.0"
+version = "2.0.0"
 
 [deps]
 Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+PyPreferences = "cc9521c6-0242-4dda-8d66-c47a9d9eec02"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289"
 
 [compat]
-Conda = "1.0"
+Conda = "1"
 MacroTools = "0.4, 0.5"
-VersionParsing = "1.0"
-julia = "0.7, 1.0"
+VersionParsing = "1"
+julia = "1.6"
 
 [extras]
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test"]
+test = ["Test", "Preferences"]

PyPreferences.jl/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Takafumi Arakaki <aka.tkf@gmail.com> and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

PyPreferences.jl/Project.toml

@@ -0,0 +1,24 @@
+name = "PyPreferences"
+uuid = "cc9521c6-0242-4dda-8d66-c47a9d9eec02"
+authors = ["Takafumi Arakaki <aka.tkf@gmail.com> and contributors"]
+version = "0.1.0"
+
+[deps]
+Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
+VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289"
+
+[compat]
+Conda = "1"
+julia = "1.6"
+VersionParsing = "1"
+Preferences = "1"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]

PyPreferences.jl/README.md

@@ -0,0 +1,8 @@
+# PyPreferences
+
+```julia
+using PyPreferences
+
+PyPreferences.use_system("python_exe")
+PyPreferences.use_conda()
+```

PyPreferences.jl/src/PyPreferences.jl

@@ -0,0 +1,41 @@
+baremodule PyPreferences
+
+using Logging
+
+function use_system end
+function use_conda end
+# function use_jll end
+function use_inprocess end
+function recompile end
+
+# API to be used from PyCall
+function assert_configured end
+function instruction_message end
+
+# function diagnose end
+function status end
+
+module Implementations
+
+    module PythonUtils
+        include("python_utils.jl")
+    end
+    include("which.jl")
+    include("core.jl")
+    include("api.jl")
+end
+
+let prefs = Implementations.setup_non_failing()
+    global const python = prefs.python
+    global const inprocess = prefs.inprocess
+    global const conda = prefs.conda
+    global const python_fullpath = prefs.python_fullpath
+    global const libpython = prefs.libpython
+    global const python_version = prefs.python_version
+    global const PYTHONHOME = prefs.PYTHONHOME
+end
+
+const pyprogramname = python_fullpath
+const pyversion_build = python_version
+
+end  # baremodule PyPreferences

PyPreferences.jl/src/api.jl

@@ -0,0 +1,62 @@
+using Conda
+
+function PyPreferences.status()
+    # TODO: compare with in-process values
+    code = """
+    $(load_pypreferences_code())
+    PyPreferences.Implementations.status_inprocess()
+    """
+    cmd = include_stdin_cmd()
+    open(pipeline(cmd; stdout = stdout, stderr = stderr); write = true) do io
+        write(io, code)
+    end
+    return
+end
+
+function PyPreferences.use_system(python::AbstractString = "python3")
+    """
+    Use python from a provided executable or path (defaults to `python3`).
+    """
+    return Implementations.set(python = python)
+end
+
+function PyPreferences.use_conda()
+    """
+    Use Python provided by Conda.jl
+    """
+    Conda.add("numpy")
+    return Implementations.set(conda = true)
+end
+
+#=
+function use_jll()
+end
+=#
+
+function PyPreferences.use_inprocess()
+    return Implementations.set(inprocess = true)
+end
+
+function PyPreferences.instruction_message()
+    return """
+    PyPreferences.jl is not configured properly. Run:
+        using Pkg
+        Pkg.add("PyPreferences")
+        using PyPreferences
+        @doc PyPreferences
+    for usage.
+    """
+end
+
+
+function PyPreferences.assert_configured()
+    if (
+        PyPreferences.python === nothing ||
+        PyPreferences.python_fullpath === nothing ||
+        PyPreferences.libpython === nothing ||
+        PyPreferences.python_version === nothing ||
+        PyPreferences.PYTHONHOME === nothing
+    )
+        error(PyPreferences.instruction_message())
+    end
+end

PyPreferences.jl/src/core.jl

@@ -0,0 +1,186 @@
+using ..PyPreferences: PyPreferences
+using .PythonUtils: find_libpython, python_version_of, pythonhome_of, conda_python_fullpath
+
+using Preferences: @set_preferences!, @load_preference, @delete_preferences!
+
+struct PythonPreferences
+    python::Union{Nothing,String}
+    inprocess::Bool
+    conda::Bool
+    # jll::Bool
+end
+
+function Base.show(io::IO,x::PythonPreferences)
+    print(io, "PythonPreferences(python=$(x.python), inprocess=$(x.inprocess), conda=$(x.conda))")
+end
+
+
+set(; python = nothing, inprocess = false, conda = false) =
+    set(PythonPreferences(python, inprocess, conda))
+
+function set(prefs::PythonPreferences)
+    @debug "setting new Python Preferences" prefs
+    if prefs.python === nothing
+        @delete_preferences!("python")
+    else
+        @set_preferences!("python" => prefs.python)
+    end
+    if prefs.inprocess
+        @set_preferences!("inprocess" => prefs.inprocess)
+    else
+        @delete_preferences!("inprocess")
+    end
+    if prefs.conda
+        @set_preferences!("conda" => prefs.conda)
+    else
+        @delete_preferences!("conda")
+    end
+    PyPreferences.recompile()
+    return prefs
+end
+
+PythonPreferences(rawprefs::AbstractDict) = PythonPreferences(
+    get(rawprefs, "python", nothing),
+    get(rawprefs, "inprocess", false),
+    get(rawprefs, "conda", false),
+)
+
+function _load_python_preferences()
+    # TODO: lookup v#.#?
+    _python = @load_preference("python", nothing)
+    _inprocess = @load_preference("inprocess", false)
+    _conda = @load_preference("conda", false)
+    #isempty(rawprefs) && return nothing
+
+    # default value
+    # if !_inprocess && !_conda && _python === nothing
+    #     _python = get_python_fullpath(get_default_python())
+    #     @info "Setting default Python interpreter to $(_python)"
+    #     return set(python=_python)
+    # end
+    return PythonPreferences(_python, _inprocess, _conda)
+end
+
+function load_pypreferences_code()
+    return """
+     $(Base.load_path_setup_code())
+     PyPreferences = Base.require(Base.PkgId(
+         Base.UUID("cc9521c6-0242-4dda-8d66-c47a9d9eec02"),
+         "PyPreferences",
+     ))
+     """
+end
+
+function include_stdin_cmd()
+    return ```
+    $(Base.julia_cmd())
+    --startup-file=no
+    -e "include_string(Main, read(stdin, String))"
+    ```
+end
+
+function PyPreferences.recompile()
+    code = """
+    $(load_pypreferences_code())
+    PyPreferences.assert_configured()
+    """
+    cmd = include_stdin_cmd()
+    open(cmd; write = true) do io
+        write(io, code)
+    end
+    return
+end
+
+"""
+Returns the default python executable used by PyCall. This defaults to
+`python3`, and can be overridden by `ENV["PYTHON"]` if it is desired.
+"""
+get_default_python() = get(ENV,"PYTHON", "python3")
+
+function get_python_fullpath(python)
+    python_fullpath = nothing
+    if python !== nothing
+        python_fullpath = _which(python)
+        if python_fullpath === nothing
+            @error "Failed to find a binary named `$(python)` in PATH."
+        else
+            @debug "Found path for command $(python)" python_fullpath
+        end
+    end
+    return python_fullpath
+end
+
+function setup_non_failing()
+    python = nothing
+    inprocess = false
+    conda = false
+    python_fullpath = nothing
+    libpython = nothing
+    python_version = nothing
+    PYTHONHOME = nothing
+
+    prefs = _load_python_preferences()
+    @debug "Loaded python preferences" prefs
+    python = prefs.python
+    inprocess = prefs.inprocess
+    conda = prefs.conda
+
+    if !inprocess 
+        if conda
+            python = python_fullpath = conda_python_fullpath()
+        elseif python === nothing
+            python = get_default_python()
+        end
+
+        @debug "Python binary selected. Attempting to find the path" python 
+
+        try
+            if python !== nothing
+                python_fullpath = _which(python)
+                if python_fullpath === nothing
+                    @error "Failed to find a binary named `$(python)` in PATH."
+                else
+                    @debug "Found path for command $(python)" python_fullpath
+                end
+            end
+
+            if python_fullpath !== nothing
+                libpython, = find_libpython(python_fullpath)
+                python_version = python_version_of(python_fullpath)
+                PYTHONHOME = pythonhome_of(python_fullpath)
+            end
+        catch err
+            @error(
+                "Failed to configure for `$python`",
+                exception = (err, catch_backtrace())
+            )
+        end
+
+        @debug "Determined python binary path" python_fullpath libpython python_version PYTHONHOME
+    end
+    if python === nothing
+        python = python_fullpath
+    end
+
+    return (
+        python = python,
+        inprocess = inprocess,
+        conda = conda,
+        python_fullpath = python_fullpath,
+        libpython = libpython,
+        python_version = python_version,
+        PYTHONHOME = PYTHONHOME,
+    )
+end
+
+function status_inprocess()
+    print("""
+    python         : $(PyPreferences.python)
+    inprocess      : $(PyPreferences.inprocess)
+    conda          : $(PyPreferences.conda)
+    python_fullpath: $(PyPreferences.python_fullpath)
+    libpython      : $(PyPreferences.libpython)
+    python_version : $(PyPreferences.python_version)
+    PYTHONHOME     : $(PyPreferences.PYTHONHOME)
+    """)
+end

deps/find_libpython.py → PyPreferences.jl/src/find_libpython.py

@@ -27,14 +27,14 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-from __future__ import print_function, absolute_import
+from __future__ import absolute_import, print_function
 
-from logging import getLogger
 import ctypes.util
 import functools
 import os
 import sys
 import sysconfig
+from logging import getLogger  # see `julia.core.logger`
 
 logger = getLogger("find_libpython")
 
@@ -76,6 +76,9 @@ class Dl_info(ctypes.Structure):
     ]
 
 
+# fmt: off
+
+
 def _linked_libpython_unix():
     libdl = ctypes.CDLL(ctypes.util.find_library("dl"))
     libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)]
@@ -88,6 +91,8 @@ def _linked_libpython_unix():
     if retcode == 0:  # means error
         return None
     path = os.path.realpath(dlinfo.dli_fname.decode())
+    if not os.path.exists(path):
+        return None
     if path == os.path.realpath(sys.executable):
         return None
     return path
@@ -182,9 +187,8 @@ def candidate_names(suffix=SHLIB_SUFFIX):
     sysdata = dict(
         v=sys.version_info,
         # VERSION is X.Y in Linux/macOS and XY in Windows:
-        VERSION=(sysconfig.get_python_version() or
-                 "{v.major}.{v.minor}".format(v=sys.version_info) or
-                 sysconfig.get_config_var("VERSION")),
+        VERSION=(sysconfig.get_config_var("VERSION") or
+                 "{v.major}.{v.minor}".format(v=sys.version_info)),
         ABIFLAGS=(sysconfig.get_config_var("ABIFLAGS") or
                   sysconfig.get_config_var("abiflags") or ""),
     )
@@ -264,8 +268,13 @@ def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple):
 
     Parameters
     ----------
-    path : str ot None
+    path : str or None
         A candidate path to a shared library.
+
+    Returns
+    -------
+    path : str or None
+        Normalized existing path or `None`.
     """
     if not path:
         return None
@@ -392,4 +401,4 @@ def main(args=None):
 
 
 if __name__ == "__main__":
-    main()
+    main()

PyPreferences.jl/src/python_utils.jl

@@ -0,0 +1,157 @@
+using Libdl: Libdl
+using VersionParsing: vparse
+using Conda: Conda
+
+conda_python_fullpath() =
+    abspath(Conda.PYTHONDIR, "python" * (Sys.iswindows() ? ".exe" : ""))
+
+# Fix the environment for running `python`, and setts IO encoding to UTF-8.
+# If cmd is the Conda python, then additionally removes all PYTHON* and
+# CONDA* environment variables.
+function pythonenv(cmd::Cmd)
+    @assert cmd.env === nothing  # TODO: handle non-nothing case
+    env = copy(ENV)
+    if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR)
+        pythonvars = String[]
+        for var in keys(env)
+            if startswith(var, "CONDA") || startswith(var, "PYTHON")
+                push!(pythonvars, var)
+            end
+        end
+        for var in pythonvars
+            pop!(env, var)
+        end
+    end
+
+    # set PYTHONIOENCODING when running python executable, so that
+    # we get UTF-8 encoded text as output (this is not the default on Windows).
+    env["PYTHONIOENCODING"] = "UTF-8"
+    setenv(cmd, env)
+end
+
+pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) =
+    chomp(read(pythonenv(`$python -c "import $mod; print($mod.$(var))"`), String))
+
+pyconfigvar(python::AbstractString, var::AbstractString) =
+    pyvar(python, "distutils.sysconfig", "get_config_var('$(var)')")
+pyconfigvar(python, var, default) =
+    let v = pyconfigvar(python, var)
+        v == "None" ? default : v
+    end
+
+function pythonhome_of(pyprogramname::AbstractString)
+    if Sys.iswindows()
+        # PYTHONHOME tells python where to look for both pure python
+        # and binary modules.  When it is set, it replaces both
+        # `prefix` and `exec_prefix` and we thus need to set it to
+        # both in case they differ. This is also what the
+        # documentation recommends.  However, they are documented
+        # to always be the same on Windows, where it causes
+        # problems if we try to include both.
+        script = """
+        import sys
+        if hasattr(sys, "base_exec_prefix"):
+            sys.stdout.write(sys.base_exec_prefix)
+        else:
+            sys.stdout.write(sys.exec_prefix)
+        """
+    else
+        script = """
+        import sys
+        if hasattr(sys, "base_exec_prefix"):
+            sys.stdout.write(sys.base_prefix)
+            sys.stdout.write(":")
+            sys.stdout.write(sys.base_exec_prefix)
+        else:
+            sys.stdout.write(sys.prefix)
+            sys.stdout.write(":")
+            sys.stdout.write(sys.exec_prefix)
+        """
+        # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME
+    end
+    return read(pythonenv(`$pyprogramname -c $script`), String)
+end
+# To support `venv` standard library (as well as `virtualenv`), we
+# need to use `sys.base_prefix` and `sys.base_exec_prefix` here.
+# Otherwise, initializing Python in `__init__` below fails with
+# unrecoverable error:
+#
+#   Fatal Python error: initfsencoding: unable to load the file system codec
+#   ModuleNotFoundError: No module named 'encodings'
+#
+# This is because `venv` does not symlink standard libraries like
+# `virtualenv`.  For example, `lib/python3.X/encodings` does not
+# exist.  Rather, `venv` relies on the behavior of Python runtime:
+#
+#   If a file named "pyvenv.cfg" exists one directory above
+#   sys.executable, sys.prefix and sys.exec_prefix are set to that
+#   directory and it is also checked for site-packages
+#   --- https://docs.python.org/3/library/venv.html
+#
+# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and
+# `sys.base_exec_prefix`.  If the virtual environment is created by
+# `virtualenv`, those `sys.base_*` paths point to the virtual
+# environment.  Thus, above code supports both use cases.
+#
+# See also:
+# * https://docs.python.org/3/library/venv.html
+# * https://docs.python.org/3/library/site.html
+# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix
+# * https://github.com/JuliaPy/PyCall.jl/issues/410
+
+python_version_of(python) = vparse(pyvar(python, "platform", "python_version()"))
+
+function find_libpython_py_path()
+
+    return joinpath(@__DIR__, "find_libpython.py")
+end
+
+function exec_find_libpython(python::AbstractString, options, verbose::Bool)
+    # Do not inline `@__DIR__` into the backticks to expand correctly.
+    # See: https://github.com/JuliaLang/julia/issues/26323
+    script = find_libpython_py_path()
+    cmd = `$python $script $options`
+    if verbose
+        cmd = `$cmd --verbose`
+    end
+    return readlines(pythonenv(cmd))
+end
+
+# return libpython path, libpython pointer
+function find_libpython(
+    python::AbstractString;
+    _dlopen = Libdl.dlopen,
+    verbose::Bool = false,
+)
+    dlopen_flags = Libdl.RTLD_LAZY | Libdl.RTLD_DEEPBIND | Libdl.RTLD_GLOBAL
+
+    libpaths = exec_find_libpython(python, `--list-all`, verbose)
+    for lib in libpaths
+        try
+            return (lib, _dlopen(lib, dlopen_flags))
+        catch e
+            @warn "Failed to `dlopen` $lib" exception = (e, catch_backtrace())
+        end
+    end
+    @warn """
+    Python (`find_libpython.py`) failed to find `libpython`.
+    Falling back to `Libdl`-based discovery.
+    """
+
+    # Try all candidate libpython names and let Libdl find the path.
+    # We do this *last* because the libpython in the system
+    # library path might be the wrong one if multiple python
+    # versions are installed (we prefer the one in LIBDIR):
+    libs = exec_find_libpython(python, `--candidate-names`, verbose)
+    for lib in libs
+        lib = splitext(lib)[1]
+        try
+            libpython = _dlopen(lib, dlopen_flags)
+            return (Libdl.dlpath(libpython), libpython)
+        catch e
+            @debug "Failed to `dlopen` $lib" exception = (e, catch_backtrace())
+        end
+    end
+
+    return nothing, nothing
+end

PyPreferences.jl/src/which.jl

@@ -0,0 +1,65 @@
+# This file vendors the utility `Sys.which` for Julia versions prior to 1.7,
+# as in 1.6 it called `realpath` and therefore breaks virtual environment
+# detection. (Without this, all `test/test_venv.jl` would fail).
+
+@static if VERSION >= v"1.7.0"
+    const _which = Sys.which
+else
+    function _which(program_name::String)
+        if isempty(program_name)
+           return nothing
+        end
+        # Build a list of program names that we're going to try
+        program_names = String[]
+        base_pname = basename(program_name)
+        if Sys.iswindows()
+            # If the file already has an extension, try that name first
+            if !isempty(splitext(base_pname)[2])
+                push!(program_names, base_pname)
+            end
+
+            # But also try appending .exe and .com`
+            for pe in (".exe", ".com")
+                push!(program_names, string(base_pname, pe))
+            end
+        else
+            # On non-windows, we just always search for what we've been given
+            push!(program_names, base_pname)
+        end
+
+        path_dirs = String[]
+        program_dirname = dirname(program_name)
+        # If we've been given a path that has a directory name in it, then we
+        # check to see if that path exists.  Otherwise, we search the PATH.
+        if isempty(program_dirname)
+            # If we have been given just a program name (not a relative or absolute
+            # path) then we should search `PATH` for it here:
+            pathsep = Sys.iswindows() ? ';' : ':'
+            path_dirs = abspath.(split(get(ENV, "PATH", ""), pathsep))
+
+            # On windows we always check the current directory as well
+            if Sys.iswindows()
+                pushfirst!(path_dirs, pwd())
+            end
+        else
+            push!(path_dirs, abspath(program_dirname))
+        end
+
+        # Here we combine our directories with our program names, searching for the
+        # first match among all combinations.
+        for path_dir in path_dirs
+            for pname in program_names
+                program_path = joinpath(path_dir, pname)
+                # If we find something that matches our name and we can execute
+                if isfile(program_path) && Sys.isexecutable(program_path)
+                    return program_path
+                end
+            end
+        end
+
+        # If we couldn't find anything, don't return anything
+        nothing
+    end
+
+    _which(program_name::AbstractString) = _which(String(program_name))
+end

PyPreferences.jl/test/runtests.jl

@@ -0,0 +1,10 @@
+using PyPreferences
+using Test
+
+@testset "PyPreferences.jl" begin
+    # Write your tests here.
+end
+
+if lowercase(get(ENV, "JULIA_PKGEVAL", "false")) != "true"
+    include("test_venv.jl")
+end

PyPreferences.jl/test/test_venv.jl

@@ -0,0 +1,123 @@
+using PyCall, Test
+
+
+function test_venv_has_python(path)
+    newpython = PyCall.python_cmd(venv=path).exec[1]
+    if !isfile(newpython)
+        @info """
+        Python executable $newpython does not exists.
+        This directory contains only the following files:
+        $(join(readdir(dirname(newpython)), '\n'))
+        """
+    end
+    @test isfile(newpython)
+end
+
+
+function test_venv_activation(path)
+    newpython = PyCall.python_cmd(venv=path).exec[1]
+
+    # Run a fresh Julia process with new Python environment
+    code = """
+    $(Base.load_path_setup_code())
+    using PyCall
+    println(PyCall.pyimport("sys").executable)
+    println(PyCall.pyimport("sys").exec_prefix)
+    println(PyCall.pyimport("pip").__file__)
+    """
+    # Note that `pip` is just some arbitrary non-standard
+    # library.  Using standard library like `os` does not work
+    # because those files are not created.
+    env = copy(ENV)
+    env["PYCALL_JL_RUNTIME_PYTHON"] = newpython
+    jlcmd = setenv(`$(Base.julia_cmd()) --startup-file=no -e $code`, env)
+    if Sys.iswindows()
+        # Marking the test broken in Windows.  It seems that
+        # venv copies .dll on Windows and libpython check in
+        # PyCall.__init__ detects that.
+        @test_broken begin
+            output = read(jlcmd, String)
+            sys_executable, exec_prefix, mod_file = split(output, "\n")
+            newpython == sys_executable
+        end
+    else
+        output = read(jlcmd, String)
+        sys_executable, exec_prefix, mod_file = split(output, "\n")
+        @test newpython == sys_executable
+        @test startswith(exec_prefix, path)
+        @test startswith(mod_file, path)
+    end
+end
+
+
+@testset "virtualenv activation" begin
+    pyname = "python$(pyversion.major).$(pyversion.minor)"
+    if Sys.which("virtualenv") === nothing
+        @info "No virtualenv command. Skipping the test..."
+    elseif Sys.which(pyname) === nothing
+        @info "No $pyname command. Skipping the test..."
+    else
+        mktempdir() do tmppath
+            if PyCall.pyversion.major == 2
+                path = joinpath(tmppath, "kind")
+            else
+                path = joinpath(tmppath, "ϵνιℓ")
+            end
+            run(`virtualenv --python=$pyname $path`)
+            test_venv_has_python(path)
+
+            newpython = PyCall.python_cmd(venv=path).exec[1]
+            venv_libpython = PyCall.find_libpython(newpython)
+            if venv_libpython != PyCall.libpython
+                @info """
+                virtualenv created an environment with incompatible libpython:
+                    $venv_libpython
+                """
+                return
+            end
+
+            test_venv_activation(path)
+        end
+    end
+end
+
+
+@testset "venv activation" begin
+    # In case PyCall is built with a Python executable created by
+    # `virtualenv`, let's try to find the original Python executable.
+    # Otherwise, `venv` does not work with this Python executable:
+    # https://bugs.python.org/issue30811
+    sys = PyCall.pyimport("sys")
+    if hasproperty(sys, :real_prefix)
+        # sys.real_prefix is set by virtualenv and does not exist in
+        # standard Python:
+        # https://github.com/pypa/virtualenv/blob/16.0.0/virtualenv_embedded/site.py#L554
+        candidates = [
+            PyCall.venv_python(sys.real_prefix, "$(pyversion.major).$(pyversion.minor)"),
+            PyCall.venv_python(sys.real_prefix, "$(pyversion.major)"),
+            PyCall.venv_python(sys.real_prefix),
+            PyCall.pyprogramname,  # must exists
+        ]
+        python = candidates[findfirst(isfile, candidates)]
+    else
+        python = PyCall.pyprogramname
+    end
+
+    if PyCall.conda
+        @info "Skip venv test with conda."
+    elseif !success(PyCall.python_cmd(`-c "import venv"`, python=python))
+        @info "Skip venv test since venv package is missing."
+    else
+        mktempdir() do tmppath
+            if PyCall.pyversion.major == 2
+                path = joinpath(tmppath, "kind")
+            else
+                path = joinpath(tmppath, "ϵνιℓ")
+            end
+            # Create a new virtual environment
+            run(PyCall.python_cmd(`-m venv $path`, python=python))
+            test_venv_has_python(path)
+            test_venv_activation(path)
+        end
+    end
+end

aot/compile.jl

@@ -9,8 +9,8 @@ using PackageCompiler
 using Pkg
 
 Pkg.activate(@__DIR__)
+Pkg.develop(PackageSpec(name="PyPreferences", path=joinpath(dirname(@__DIR__), "PyPreferences.jl")))
 Pkg.develop(PackageSpec(name="PyCall", path=dirname(@__DIR__)))
-Pkg.build("PyCall")
 Pkg.activate()
 
 sysimage_path = joinpath(@__DIR__, "sys.$(Libdl.dlext)")

src/PyCall.jl

@@ -4,7 +4,8 @@ if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@optle
     @eval Base.Experimental.@optlevel 1
 end
 
-using VersionParsing
+using PyPreferences
+using VersionParsing: vparse
 
 export pycall, pycall!, pyimport, pyimport_e, pybuiltin, PyObject, PyReverseDims,
        PyPtr, pyincref, pydecref, pyversion,
@@ -34,7 +35,7 @@ import Base.Iterators: filter
 
 #########################################################################
 
-include(joinpath(dirname(@__FILE__), "..", "deps","depsutils.jl"))
+include("startup_helpers.jl")
 include("startup.jl")
 
 """

src/pyinit.jl

@@ -98,7 +98,7 @@ function python_cmd(args::Cmd = ``;
 end
 
 function find_libpython(python::AbstractString)
-    script = joinpath(@__DIR__, "..", "deps", "find_libpython.py")
+    script = joinpath(@__DIR__, "..", "PyPreferences.jl", "src", "find_libpython.py")
     cmd = python_cmd(`$script`; python = python)
     try
         return read(cmd, String)
@@ -154,34 +154,7 @@ function __init__()
     if !already_inited
         pyhome = PYTHONHOME
 
-        if isfile(get(ENV, "PYCALL_JL_RUNTIME_PYTHON", ""))
-            _current_python[] = ENV["PYCALL_JL_RUNTIME_PYTHON"]
-
-            # Check libpython compatibility.
-            venv_libpython = find_libpython(current_python())
-            if venv_libpython === nothing
-                error("""
-                `libpython` for $(current_python()) cannot be found.
-                PyCall.jl cannot initialize Python safely.
-                """)
-            elseif venv_libpython != libpython
-                error("""
-                Incompatible `libpython` detected.
-                `libpython` for $(current_python()) is:
-                    $venv_libpython
-                `libpython` for $pyprogramname is:
-                    $libpython
-                PyCall.jl only supports loading Python environment using
-                the same `libpython`.
-                """)
-            end
-
-            if haskey(ENV, "PYCALL_JL_RUNTIME_PYTHONHOME")
-                pyhome = ENV["PYCALL_JL_RUNTIME_PYTHONHOME"]
-            else
-                pyhome = pythonhome_of(current_python())
-            end
-        elseif conda && Sys.iswindows()
+        if conda && Sys.iswindows()
             # some Python modules on Windows need the PATH to include
             # Anaconda's Library\bin directory in order to find their DLL files
             ENV["PATH"] = Conda.bin_dir(Conda.ROOTENV) * ";" * get(ENV, "PATH", "")
@@ -201,7 +174,7 @@ function __init__()
 
     if new_pyversion.major != pyversion.major
         error("PyCall precompiled with Python $pyversion, but now using Python $new_pyversion; ",
-              "you need to relaunch Julia and re-run Pkg.build(\"PyCall\")")
+              "you need to relaunch Julia and run `using PyPreferences; PyPreferences.recompile()")
     end
 
     copy!(inspect, pyimport("inspect"))

src/startup.jl

@@ -38,17 +38,22 @@ else
     global symbols_present = hassym(proc_handle, :Py_GetVersion)
 end
 
+if PyPreferences.inprocess
+    @assert symbols_present # TODO: better error
+end
+
 if !symbols_present
+    PyPreferences.assert_configured()
+    using PyPreferences: PYTHONHOME, conda, libpython, pyprogramname, python, pyversion_build
     # Python not present. Use deps.jl
     const depfile = joinpath(dirname(@__FILE__), "..", "deps", "deps.jl")
-    isfile(depfile) || error("PyCall not properly installed. Please run Pkg.build(\"PyCall\")")
-    include(depfile) # generated by Pkg.build("PyCall")
+
     # Only to be used at top-level - pointer will be invalid after reload
     libpy_handle = try
         Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL)
     catch err
         if err isa ErrorException
-            error(err.msg, ". Please run `Pkg.build(\"PyCall\")` if your Python build has changed")
+            error(err.msg, "\n", PyPreferences.instruction_message())
         else
             rethrow(err)
         end

deps/depsutils.jl → src/startup_helpers.jl

@@ -95,63 +95,3 @@ function pythonenv(cmd::Cmd)
     setenv(cmd, env)
 end
 
-
-function pythonhome_of(pyprogramname::AbstractString)
-    if Sys.iswindows()
-        # PYTHONHOME tells python where to look for both pure python
-        # and binary modules.  When it is set, it replaces both
-        # `prefix` and `exec_prefix` and we thus need to set it to
-        # both in case they differ. This is also what the
-        # documentation recommends.  However, they are documented
-        # to always be the same on Windows, where it causes
-        # problems if we try to include both.
-        script = """
-        import sys
-        if hasattr(sys, "base_exec_prefix"):
-            sys.stdout.write(sys.base_exec_prefix)
-        else:
-            sys.stdout.write(sys.exec_prefix)
-        """
-    else
-        script = """
-        import sys
-        if hasattr(sys, "base_exec_prefix"):
-            sys.stdout.write(sys.base_prefix)
-            sys.stdout.write(":")
-            sys.stdout.write(sys.base_exec_prefix)
-        else:
-            sys.stdout.write(sys.prefix)
-            sys.stdout.write(":")
-            sys.stdout.write(sys.exec_prefix)
-        """
-        # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME
-    end
-    return read(pythonenv(`$pyprogramname -c $script`), String)
-end
-# To support `venv` standard library (as well as `virtualenv`), we
-# need to use `sys.base_prefix` and `sys.base_exec_prefix` here.
-# Otherwise, initializing Python in `__init__` below fails with
-# unrecoverable error:
-#
-#   Fatal Python error: initfsencoding: unable to load the file system codec
-#   ModuleNotFoundError: No module named 'encodings'
-#
-# This is because `venv` does not symlink standard libraries like
-# `virtualenv`.  For example, `lib/python3.X/encodings` does not
-# exist.  Rather, `venv` relies on the behavior of Python runtime:
-#
-#   If a file named "pyvenv.cfg" exists one directory above
-#   sys.executable, sys.prefix and sys.exec_prefix are set to that
-#   directory and it is also checked for site-packages
-#   --- https://docs.python.org/3/library/venv.html
-#
-# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and
-# `sys.base_exec_prefix`.  If the virtual environment is created by
-# `virtualenv`, those `sys.base_*` paths point to the virtual
-# environment.  Thus, above code supports both use cases.
-#
-# See also:
-# * https://docs.python.org/3/library/venv.html
-# * https://docs.python.org/3/library/site.html
-# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix
-# * https://github.com/JuliaPy/PyCall.jl/issues/410

test/runtests.jl

@@ -1,5 +1,17 @@
+using Preferences
+pypref_py = lowercase(get(ENV, "PYPREFERENCES_PYTHON", ""))
+if pypref_py == "conda"
+    PyPrefs_UUID = Base.UUID("cc9521c6-0242-4dda-8d66-c47a9d9eec02")
+    delete_preferences!(PyPrefs_UUID, "python3", force=true)
+    set_preferences!(PyPrefs_UUID, "conda"=>true)
+
+    using PyPreferences
+    PyPreferences.use_conda()
+end
+
 using PyCall
 using PyCall: hasproperty
+using PyPreferences
 using Test, Dates, Serialization
 
 filter(f, itr) = collect(Iterators.filter(f, itr))
@@ -8,6 +20,9 @@ filter(f, d::AbstractDict) = Base.filter(f, d)
 PYTHONPATH=get(ENV,"PYTHONPATH","")
 PYTHONHOME=get(ENV,"PYTHONHOME","")
 PYTHONEXECUTABLE=get(ENV,"PYTHONEXECUTABLE","")
+
+PyPreferences.status()
+
 @info "Python version $pyversion from $(PyCall.libpython), PYTHONHOME=$(PyCall.PYTHONHOME)\nENV[PYTHONPATH]=$PYTHONPATH\nENV[PYTHONHOME]=$PYTHONHOME\nENV[PYTHONEXECUTABLE]=$PYTHONEXECUTABLE"
 
 @testset "CI setup" begin
@@ -16,6 +31,8 @@ PYTHONEXECUTABLE=get(ENV,"PYTHONEXECUTABLE","")
     end
 end
 
+include("test_venv.jl")
+
 roundtrip(T, x) = convert(T, PyObject(x))
 roundtrip(x) = roundtrip(PyAny, x)
 roundtripeq(T, x) = roundtrip(T, x) == x
@@ -817,7 +834,6 @@ include("test_pyfncall.jl")
 include("testpybuffer.jl")
 if lowercase(get(ENV, "JULIA_PKGEVAL", "false")) != "true"
     include("test_venv.jl")
-    include("test_build.jl")
 end
 
 @testset "@pyinclude" begin

test/test_venv.jl

@@ -18,8 +18,14 @@ function test_venv_activation(path)
     newpython = PyCall.python_cmd(venv=path).exec[1]
 
     # Run a fresh Julia process with new Python environment
+    # must manually set the preferences before loading PyPreferences
+    # in order not to have to run again the julia process 
     code = """
     $(Base.load_path_setup_code())
+    using Preferences
+    PyPrefs_UUID = Base.UUID("cc9521c6-0242-4dda-8d66-c47a9d9eec02")
+    delete_preferences!(PyPrefs_UUID, "conda", force=true)
+    set_preferences!(PyPrefs_UUID, "python"=>"$newpython", force=true)
     using PyCall
     println(PyCall.pyimport("sys").executable)
     println(PyCall.pyimport("sys").exec_prefix)
@@ -29,7 +35,6 @@ function test_venv_activation(path)
     # library.  Using standard library like `os` does not work
     # because those files are not created.
     env = copy(ENV)
-    env["PYCALL_JL_RUNTIME_PYTHON"] = newpython
     jlcmd = setenv(`$(Base.julia_cmd()) --startup-file=no -e $code`, env)
     if Sys.iswindows()
         # Marking the test broken in Windows.  It seems that