Skip to content

Commit

Permalink
Test find_libpython (JuliaPy#595)
Browse files Browse the repository at this point in the history
  • Loading branch information
tkf authored and stevengj committed Feb 27, 2019
1 parent 4ab3d44 commit 79d5141
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 81 deletions.
82 changes: 1 addition & 81 deletions deps/build.jl
Original file line number Diff line number Diff line change
Expand Up @@ -10,87 +10,7 @@ import Conda, Libdl

struct UseCondaPython <: Exception end

#########################################################################

pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) = chomp(read(pythonenv(`$python -c "import $mod; print($mod.$var)"`), String))

pyconfigvar(python::AbstractString, var::AbstractString) = pyvar(python, "distutils.sysconfig", "get_config_var('$var')")
pyconfigvar(python, var, default) = let v = pyconfigvar(python, var)
v == "None" ? default : v
end

pysys(python::AbstractString, var::AbstractString) = pyvar(python, "sys", var)

#########################################################################

const dlprefix = Sys.iswindows() ? "" : "lib"

# print out extra info to help with remote debugging
const PYCALL_DEBUG_BUILD = "yes" == get(ENV, "PYCALL_DEBUG_BUILD", "no")

function exec_find_libpython(python::AbstractString, options)
# Do not inline `@__DIR__` into the backticks to expand correctly.
# See: https://github.com/JuliaLang/julia/issues/26323
script = joinpath(@__DIR__, "find_libpython.py")
cmd = `$python $script $options`
if PYCALL_DEBUG_BUILD
cmd = `$cmd --verbose`
end
return readlines(pythonenv(cmd))
end

function show_dlopen_error(lib, e)
if PYCALL_DEBUG_BUILD
println(stderr, "dlopen($lib) ==> ", e)
# Using STDERR since find_libpython.py prints debugging
# messages to STDERR too.
end
end

# return libpython name, libpython pointer
function find_libpython(python::AbstractString)
dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL

libpaths = exec_find_libpython(python, `--list-all`)
for lib in libpaths
try
return (Libdl.dlopen(lib, dlopen_flags), lib)
catch e
show_dlopen_error(lib, e)
end
end

# Try all candidate libpython names and let Libdl find the path.
# We do this *last* because the libpython in the system
# library path might be the wrong one if multiple python
# versions are installed (we prefer the one in LIBDIR):
libs = exec_find_libpython(python, `--candidate-names`)
for lib in libs
lib = splitext(lib)[1]
try
libpython = Libdl.dlopen(lib, dlopen_flags)
# Store the fullpath to libpython in deps.jl. This makes
# it easier for users to investigate Python setup
# PyCall.jl trying to use. It also helps PyJulia to
# compare libpython.
return (libpython, Libdl.dlpath(libpython))
catch e
show_dlopen_error(lib, e)
end
end

error("""
Couldn't find libpython; check your PYTHON environment variable.
The python executable we tried was $python.
Re-building with
ENV["PYCALL_DEBUG_BUILD"] = "yes"
may provide extra information for why it failed.
""")
end

#########################################################################

include("buildutils.jl")
include("depsutils.jl")

#########################################################################
Expand Down
80 changes: 80 additions & 0 deletions deps/buildutils.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# Included from build.jl and ../test/test_build.jl

using VersionParsing
import Conda, Libdl

pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) = chomp(read(pythonenv(`$python -c "import $mod; print($mod.$var)"`), String))

pyconfigvar(python::AbstractString, var::AbstractString) = pyvar(python, "distutils.sysconfig", "get_config_var('$var')")
pyconfigvar(python, var, default) = let v = pyconfigvar(python, var)
v == "None" ? default : v
end

pysys(python::AbstractString, var::AbstractString) = pyvar(python, "sys", var)

#########################################################################

# print out extra info to help with remote debugging
const PYCALL_DEBUG_BUILD = "yes" == get(ENV, "PYCALL_DEBUG_BUILD", "no")

function exec_find_libpython(python::AbstractString, options)
# Do not inline `@__DIR__` into the backticks to expand correctly.
# See: https://github.com/JuliaLang/julia/issues/26323
script = joinpath(@__DIR__, "find_libpython.py")
cmd = `$python $script $options`
if PYCALL_DEBUG_BUILD
cmd = `$cmd --verbose`
end
return readlines(pythonenv(cmd))
end

function show_dlopen_error(lib, e)
if PYCALL_DEBUG_BUILD
println(stderr, "dlopen($lib) ==> ", e)
# Using STDERR since find_libpython.py prints debugging
# messages to STDERR too.
end
end

# return libpython name, libpython pointer
function find_libpython(python::AbstractString; _dlopen = Libdl.dlopen)
dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL

libpaths = exec_find_libpython(python, `--list-all`)
for lib in libpaths
try
return (_dlopen(lib, dlopen_flags), lib)
catch e
show_dlopen_error(lib, e)
end
end

# Try all candidate libpython names and let Libdl find the path.
# We do this *last* because the libpython in the system
# library path might be the wrong one if multiple python
# versions are installed (we prefer the one in LIBDIR):
libs = exec_find_libpython(python, `--candidate-names`)
for lib in libs
lib = splitext(lib)[1]
try
libpython = _dlopen(lib, dlopen_flags)
# Store the fullpath to libpython in deps.jl. This makes
# it easier for users to investigate Python setup
# PyCall.jl trying to use. It also helps PyJulia to
# compare libpython.
return (libpython, Libdl.dlpath(libpython))
catch e
show_dlopen_error(lib, e)
end
end

v = pyconfigvar(python, "VERSION", "unknown")
error("""
Couldn't find libpython; check your PYTHON environment variable.
The python executable we tried was $python (= version $v).
Re-building with
ENV["PYCALL_DEBUG_BUILD"] = "yes"
may provide extra information for why it failed.
""")
end
2 changes: 2 additions & 0 deletions deps/depsutils.jl
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# Included from build.jl, ../test/test_build.jl and ../src/PyCall.jl

import Libdl

hassym(lib, sym) = Libdl.dlsym_e(lib, sym) != C_NULL
Expand Down
1 change: 1 addition & 0 deletions test/runtests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -770,3 +770,4 @@ end
include("test_pyfncall.jl")
include("testpybuffer.jl")
include("test_venv.jl")
include("test_build.jl")
49 changes: 49 additions & 0 deletions test/test_build.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
module TestPyCallBuild

include(joinpath(dirname(@__FILE__), "..", "deps", "depsutils.jl"))
include(joinpath(dirname(@__FILE__), "..", "deps", "buildutils.jl"))

using Test

@testset "find_libpython" begin
for python in ["python", "python2", "python3"]
if Sys.which(python) === nothing
@info "$python not available; skipping test"
else
@test isfile(find_libpython(python)[2])
end
end

# Test the case `find_libpython.py` does not print anything. We
# use the command `true` to mimic this case.
if Sys.which("true") === nothing
@info "no `true` command; skipping test"
else
let err, msg
@test try
find_libpython("true")
false
catch err
err isa ErrorException
end
msg = sprint(showerror, err)
@test occursin("Couldn't find libpython", msg)
@test occursin("ENV[\"PYCALL_DEBUG_BUILD\"] = \"yes\"", msg)
end
end

# Test the case `dlopen` failed to open the library.
let err, msg
@test try
find_libpython("python"; _dlopen = (_...) -> error("dummy"))
false
catch err
err isa ErrorException
end
msg = sprint(showerror, err)
@test occursin("Couldn't find libpython", msg)
@test occursin("ENV[\"PYCALL_DEBUG_BUILD\"] = \"yes\"", msg)
end
end

end # module

0 comments on commit 79d5141

Please sign in to comment.