Merge remote-tracking branch 'origin/main' into v1

Author: Christopher Doris

docs/src/compat.md

@@ -9,6 +9,12 @@ functions to bridge a gap. We aim to keep these as minimal as possible.
 
 Whenever a Python exception is displayed by Julia, `sys.last_traceback` and friends are set. This allows the post-mortem debugger `pdb.pm()` to work. Disable by setting `PythonCall.CONFIG.auto_sys_last_traceback = false`.
 
+## Julia standard library
+
+Python objects can be serialised with the [`Serialization`](https://docs.julialang.org/en/v1/stdlib/Serialization/) stdlib.
+This uses [`pickle`](https://docs.python.org/3/library/pickle.html) library under the hood.
+You can opt into using [`dill`](https://pypi.org/project/dill/) instead by setting the environment variable `JULIA_PYTHONCALL_PICKLE="dill"`.
+
 ## Tabular data / Pandas
 
 The abstract type [`PyTable`](@ref) is for wrapper types around Python tables, providing the

docs/src/releasenotes.md

@@ -3,6 +3,10 @@
 ## Unreleased (v1)
 * `PythonCall.GC` is now more like `Base.GC`: `enable(true)` replaces `enable()`, `enable(false)` replaces `disable()`, and `gc()` is added.
 
+## Unreleased
+* `Serialization.serialize` can use `dill` instead of `pickle` by setting the env var `JULIA_PYTHONCALL_PICKLE=dill`.
+* `datetime.timedelta` can now be converted to `Dates.Nanosecond`, `Microsecond`, `Millisecond` and `Second`. This behaviour was already documented.
+
 ## 0.9.20 (2024-05-01)
 * The IPython extension is now automatically loaded upon import if IPython is detected.
 * JuliaCall now compatible with Julia 1.10.3.

src/Compat/serialization.jl

@@ -2,11 +2,13 @@
 #
 # We use pickle to serialise Python objects to bytes.
 
+_pickle_module() = pyimport(get(ENV, "JULIA_PYTHONCALL_PICKLE", "pickle"))
+
 function serialize_py(s, x::Py)
     if pyisnull(x)
         serialize(s, nothing)
     else
-        b = pyimport("pickle").dumps(x)
+        b = _pickle_module().dumps(x)
         serialize(s, pybytes_asvector(b))
     end
 end
@@ -16,7 +18,7 @@ function deserialize_py(s)
     if v === nothing
         pynew()
     else
-        pyimport("pickle").loads(pybytes(v))
+        _pickle_module().loads(pybytes(v))
     end
 end
 

src/Convert/Convert.jl

@@ -7,7 +7,7 @@ module Convert
 
 using ..Core
 using ..Core: C, Utils, @autopy, getptr, incref, pynew, PyNULL, pyisnull, pydel!, pyisint, iserrset_ambig, pyisnone, pyisTrue, pyisFalse, pyfloat_asdouble, pycomplex_ascomplex, pyisstr, pystr_asstring, pyisbytes, pybytes_asvector, pybytes_asUTF8string, pyisfloat, pyisrange, pytuple_getitem, unsafe_pynext, pyistuple, pydatetimetype, pytime_isaware, pydatetime_isaware, _base_pydatetime, _base_datetime, errmatches, errclear, errset, pyiscomplex, pythrow, pybool_asbool
-using Dates: Date, Time, DateTime, Millisecond
+using Dates: Date, Time, DateTime, Second, Millisecond, Microsecond, Nanosecond
 
 import ..Core: pyconvert
 
@@ -18,7 +18,7 @@ include("numpy.jl")
 include("pandas.jl")
 
 function __init__()
-    C.with_gil() do 
+    C.with_gil() do
         init_pyconvert()
         init_ctypes()
         init_numpy()

src/Convert/pyconvert.jl

@@ -7,12 +7,12 @@
 end
 
 struct PyConvertRule
-    type :: Type
-    func :: Function
-    priority :: PyConvertPriority
+    type::Type
+    func::Function
+    priority::PyConvertPriority
 end
 
-const PYCONVERT_RULES = Dict{String, Vector{PyConvertRule}}()
+const PYCONVERT_RULES = Dict{String,Vector{PyConvertRule}}()
 const PYCONVERT_EXTRATYPES = Py[]
 
 """
@@ -201,7 +201,7 @@ function _pyconvert_get_rules(pytype::Py)
     # check the original MRO is preserved
     omro_ = filter(t -> pyisin(t, omro), mro)
     @assert length(omro) == length(omro_)
-    @assert all(pyis(x,y) for (x,y) in zip(omro, omro_))
+    @assert all(pyis(x, y) for (x, y) in zip(omro, omro_))
 
     # get the names of the types in the MRO of pytype
     xmro = [String[pyconvert_typename(t)] for t in mro]
@@ -240,22 +240,23 @@ function _pyconvert_get_rules(pytype::Py)
     rules = PyConvertRule[rule for tname in mro for rule in get!(Vector{PyConvertRule}, PYCONVERT_RULES, tname)]
 
     # order the rules by priority, then by original order
-    order = sort(axes(rules, 1), by = i -> (rules[i].priority, -i), rev = true)
+    order = sort(axes(rules, 1), by=i -> (rules[i].priority, -i), rev=true)
     rules = rules[order]
 
-    @debug "pyconvert" pytype mro=join(mro, " ")
+    @debug "pyconvert" pytype mro = join(mro, " ")
     return rules
 end
 
 const PYCONVERT_PREFERRED_TYPE = Dict{Py,Type}()
 
-pyconvert_preferred_type(pytype::Py) = get!(PYCONVERT_PREFERRED_TYPE, pytype) do
-    if pyissubclass(pytype, pybuiltins.int)
-        Union{Int,BigInt}
-    else
-        _pyconvert_get_rules(pytype)[1].type
+pyconvert_preferred_type(pytype::Py) =
+    get!(PYCONVERT_PREFERRED_TYPE, pytype) do
+        if pyissubclass(pytype, pybuiltins.int)
+            Union{Int,BigInt}
+        else
+            _pyconvert_get_rules(pytype)[1].type
+        end
     end
-end
 
 function pyconvert_get_rules(type::Type, pytype::Py)
     @nospecialize type
@@ -281,15 +282,15 @@ end
 
 pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x)
 
-const PYCONVERT_RULES_CACHE = Dict{Type, Dict{C.PyPtr, Vector{Function}}}()
+const PYCONVERT_RULES_CACHE = Dict{Type,Dict{C.PyPtr,Vector{Function}}}()
 
-@generated pyconvert_rules_cache(::Type{T}) where {T} = get!(Dict{C.PyPtr, Vector{Function}}, PYCONVERT_RULES_CACHE, T)
+@generated pyconvert_rules_cache(::Type{T}) where {T} = get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE, T)
 
 function pyconvert_rule_fast(::Type{T}, x::Py) where {T}
     if T isa Union
-        a = pyconvert_rule_fast(T.a, x) :: pyconvert_returntype(T.a)
+        a = pyconvert_rule_fast(T.a, x)::pyconvert_returntype(T.a)
         pyconvert_isunconverted(a) || return a
-        b = pyconvert_rule_fast(T.b, x) :: pyconvert_returntype(T.b)
+        b = pyconvert_rule_fast(T.b, x)::pyconvert_returntype(T.b)
         pyconvert_isunconverted(b) || return b
     elseif (T == Nothing) | (T == Missing)
         pyisnone(x) && return pyconvert_return(T())
@@ -318,7 +319,7 @@ function pytryconvert(::Type{T}, x_) where {T}
 
     # We can optimize the conversion for some types by overloading pytryconvert_fast.
     # It MUST give the same results as via the slower route using rules.
-    ans1 = pyconvert_rule_fast(T, x) :: pyconvert_returntype(T)
+    ans1 = pyconvert_rule_fast(T, x)::pyconvert_returntype(T)
     pyconvert_isunconverted(ans1) || return ans1
 
     # get rules from the cache
@@ -334,7 +335,7 @@ function pytryconvert(::Type{T}, x_) where {T}
 
     # apply the rules
     for rule in rules
-        ans2 = rule(x) :: pyconvert_returntype(T)
+        ans2 = rule(x)::pyconvert_returntype(T)
         pyconvert_isunconverted(ans2) || return ans2
     end
 
@@ -386,8 +387,8 @@ pyconvertarg(::Type{T}, x, name) where {T} = @autopy x @pyconvert T x_ begin
 end
 
 function init_pyconvert()
-    push!(PYCONVERT_EXTRATYPES, pyimport("io"=>"IOBase"))
-    push!(PYCONVERT_EXTRATYPES, pyimport("numbers"=>("Number", "Complex", "Real", "Rational", "Integral"))...)
+    push!(PYCONVERT_EXTRATYPES, pyimport("io" => "IOBase"))
+    push!(PYCONVERT_EXTRATYPES, pyimport("numbers" => ("Number", "Complex", "Real", "Rational", "Integral"))...)
     push!(PYCONVERT_EXTRATYPES, pyimport("collections.abc" => ("Iterable", "Sequence", "Set", "Mapping"))...)
 
     priority = PYCONVERT_PRIORITY_CANONICAL
@@ -405,6 +406,7 @@ function init_pyconvert()
     pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority)
     pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date, priority)
     pyconvert_add_rule("datetime:time", Time, pyconvert_rule_time, priority)
+    pyconvert_add_rule("datetime:timedelta", Microsecond, pyconvert_rule_timedelta, priority)
     pyconvert_add_rule("builtins:BaseException", PyException, pyconvert_rule_exception, priority)
 
     priority = PYCONVERT_PRIORITY_NORMAL
@@ -428,6 +430,9 @@ function init_pyconvert()
     pyconvert_add_rule("collections.abc:Sequence", Tuple, pyconvert_rule_iterable, priority)
     pyconvert_add_rule("collections.abc:Set", Set, pyconvert_rule_iterable, priority)
     pyconvert_add_rule("collections.abc:Mapping", Dict, pyconvert_rule_mapping, priority)
+    pyconvert_add_rule("datetime:timedelta", Millisecond, pyconvert_rule_timedelta, priority)
+    pyconvert_add_rule("datetime:timedelta", Second, pyconvert_rule_timedelta, priority)
+    pyconvert_add_rule("datetime:timedelta", Nanosecond, pyconvert_rule_timedelta, priority)
 
     priority = PYCONVERT_PRIORITY_FALLBACK
     pyconvert_add_rule("builtins:object", Py, pyconvert_rule_object, priority)

src/Convert/rules.jl

@@ -133,7 +133,7 @@ function pyconvert_rule_range(::Type{R}, x::Py, ::Type{StepRange{T0,S0}}=Utils._
     a′, c′ = promote(a, c - oftype(c, sign(b)))
     T2 = Utils._promote_type_bounded(T0, typeof(a′), typeof(c′), T1)
     S2 = Utils._promote_type_bounded(S0, typeof(c′), S1)
-    pyconvert_return(StepRange{T2, S2}(a′, b, c′))
+    pyconvert_return(StepRange{T2,S2}(a′, b, c′))
 end
 
 function pyconvert_rule_range(::Type{R}, x::Py, ::Type{UnitRange{T0}}=Utils._type_lb(R), ::Type{UnitRange{T1}}=Utils._type_ub(R)) where {R<:UnitRange,T0,T1}
@@ -261,7 +261,7 @@ function pyconvert_rule_iterable(::Type{T}, xs::Py) where {T<:Tuple}
     zs = Any[]
     for x in xs
         if length(zs) < length(ts)
-            t = ts[length(zs) + 1]
+            t = ts[length(zs)+1]
         elseif isvararg
             t = vartype
         else
@@ -282,7 +282,7 @@ for N in 0:16
         n = pylen(xs)
         n == $N || return pyconvert_unconverted()
         $((
-            :($z = @pyconvert($T, pytuple_getitem(xs, $(i-1))))
+            :($z = @pyconvert($T, pytuple_getitem(xs, $(i - 1))))
             for (i, T, z) in zip(1:N, Ts, zs)
         )...)
         return pyconvert_return(($(zs...),))
@@ -293,12 +293,12 @@ for N in 0:16
         n = pylen(xs)
         n ≥ $N || return pyconvert_unconverted()
         $((
-            :($z = @pyconvert($T, pytuple_getitem(xs, $(i-1))))
+            :($z = @pyconvert($T, pytuple_getitem(xs, $(i - 1))))
             for (i, T, z) in zip(1:N, Ts, zs)
         )...)
         vs = V[]
-        for i in $(N+1):n
-            v = @pyconvert(V, pytuple_getitem(xs, i-1))
+        for i in $(N + 1):n
+            v = @pyconvert(V, pytuple_getitem(xs, i - 1))
             push!(vs, v)
         end
         return pyconvert_return(($(zs...), vs...))
@@ -395,3 +395,47 @@ function pyconvert_rule_datetime(::Type{DateTime}, x::Py)
     iszero(mod(microseconds, 1000)) || return pyconvert_unconverted()
     return pyconvert_return(_base_datetime + Millisecond(div(microseconds, 1000) + 1000 * (seconds + 60 * 60 * 24 * days)))
 end
+
+function pyconvert_rule_timedelta(::Type{Nanosecond}, x::Py)
+    days = pyconvert(Int, x.days)
+    if abs(days) ≥ 106751
+        # overflow
+        return pyconvert_unconverted()
+    end
+    seconds = pyconvert(Int, x.seconds)
+    microseconds = pyconvert(Int, x.microseconds)
+    return Nanosecond(((days * 3600 * 24 + seconds) * 1000000 + microseconds) * 1000)
+end
+
+function pyconvert_rule_timedelta(::Type{Microsecond}, x::Py)
+    days = pyconvert(Int, x.days)
+    if abs(days) ≥ 106751990
+        # overflow
+        return pyconvert_unconverted()
+    end
+    seconds = pyconvert(Int, x.seconds)
+    microseconds = pyconvert(Int, x.microseconds)
+    return Microsecond((days * 3600 * 24 + seconds) * 1000000 + microseconds)
+end
+
+function pyconvert_rule_timedelta(::Type{Millisecond}, x::Py)
+    days = pyconvert(Int, x.days)
+    seconds = pyconvert(Int, x.seconds)
+    microseconds = pyconvert(Int, x.microseconds)
+    if mod(microseconds, 1000) != 0
+        # inexact
+        return pyconvert_unconverted()
+    end
+    return Millisecond((days * 3600 * 24 + seconds) * 1000 + div(microseconds, 1000))
+end
+
+function pyconvert_rule_timedelta(::Type{Second}, x::Py)
+    days = pyconvert(Int, x.days)
+    seconds = pyconvert(Int, x.seconds)
+    microseconds = pyconvert(Int, x.microseconds)
+    if microseconds != 0
+        # inexact
+        return pyconvert_unconverted()
+    end
+    return Second(days * 3600 * 24 + seconds)
+end