From 875c34d2e8ba54036b035ed26fe414e55256b8a1 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Wed, 15 May 2024 21:11:58 +0100 Subject: [PATCH] Timedelta conversion (#499) * add rules for datetime.timedelta * tests for timedelta conversion * document new timedelta conversion rules --------- Co-authored-by: Christopher Doris --- docs/src/releasenotes.md | 1 + src/Convert/Convert.jl | 4 +-- src/Convert/pyconvert.jl | 47 ++++++++++++++++++--------------- src/Convert/rules.jl | 56 +++++++++++++++++++++++++++++++++++----- test/Convert.jl | 42 ++++++++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 29 deletions(-) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 4566dbc3..dba731e8 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -2,6 +2,7 @@ ## Unreleased * `Serialization.serialize` can use `dill` instead of `pickle` by setting the env var `JULIA_PYTHONCALL_PICKLE=dill`. +* `datetime.timedelta` can now be converted to `Dates.Nanosecond`, `Microsecond`, `Millisecond` and `Second`. This behaviour was already documented. ## 0.9.20 (2024-05-01) * The IPython extension is now automatically loaded upon import if IPython is detected. diff --git a/src/Convert/Convert.jl b/src/Convert/Convert.jl index 961e84f0..0cccbf58 100644 --- a/src/Convert/Convert.jl +++ b/src/Convert/Convert.jl @@ -7,7 +7,7 @@ module Convert using ..Core using ..Core: C, Utils, @autopy, getptr, incref, pynew, PyNULL, pyisnull, pydel!, pyisint, iserrset_ambig, pyisnone, pyisTrue, pyisFalse, pyfloat_asdouble, pycomplex_ascomplex, pyisstr, pystr_asstring, pyisbytes, pybytes_asvector, pybytes_asUTF8string, pyisfloat, pyisrange, pytuple_getitem, unsafe_pynext, pyistuple, pydatetimetype, pytime_isaware, pydatetime_isaware, _base_pydatetime, _base_datetime, errmatches, errclear, errset, pyiscomplex, pythrow, pybool_asbool -using Dates: Date, Time, DateTime, Millisecond +using Dates: Date, Time, DateTime, Second, Millisecond, Microsecond, Nanosecond import ..Core: pyconvert @@ -18,7 +18,7 @@ include("numpy.jl") include("pandas.jl") function __init__() - C.with_gil() do + C.with_gil() do init_pyconvert() init_ctypes() init_numpy() diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index a7293051..9963940f 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -7,12 +7,12 @@ end struct PyConvertRule - type :: Type - func :: Function - priority :: PyConvertPriority + type::Type + func::Function + priority::PyConvertPriority end -const PYCONVERT_RULES = Dict{String, Vector{PyConvertRule}}() +const PYCONVERT_RULES = Dict{String,Vector{PyConvertRule}}() const PYCONVERT_EXTRATYPES = Py[] """ @@ -201,7 +201,7 @@ function _pyconvert_get_rules(pytype::Py) # check the original MRO is preserved omro_ = filter(t -> pyisin(t, omro), mro) @assert length(omro) == length(omro_) - @assert all(pyis(x,y) for (x,y) in zip(omro, omro_)) + @assert all(pyis(x, y) for (x, y) in zip(omro, omro_)) # get the names of the types in the MRO of pytype xmro = [String[pyconvert_typename(t)] for t in mro] @@ -240,22 +240,23 @@ function _pyconvert_get_rules(pytype::Py) rules = PyConvertRule[rule for tname in mro for rule in get!(Vector{PyConvertRule}, PYCONVERT_RULES, tname)] # order the rules by priority, then by original order - order = sort(axes(rules, 1), by = i -> (rules[i].priority, -i), rev = true) + order = sort(axes(rules, 1), by=i -> (rules[i].priority, -i), rev=true) rules = rules[order] - @debug "pyconvert" pytype mro=join(mro, " ") + @debug "pyconvert" pytype mro = join(mro, " ") return rules end const PYCONVERT_PREFERRED_TYPE = Dict{Py,Type}() -pyconvert_preferred_type(pytype::Py) = get!(PYCONVERT_PREFERRED_TYPE, pytype) do - if pyissubclass(pytype, pybuiltins.int) - Union{Int,BigInt} - else - _pyconvert_get_rules(pytype)[1].type +pyconvert_preferred_type(pytype::Py) = + get!(PYCONVERT_PREFERRED_TYPE, pytype) do + if pyissubclass(pytype, pybuiltins.int) + Union{Int,BigInt} + else + _pyconvert_get_rules(pytype)[1].type + end end -end function pyconvert_get_rules(type::Type, pytype::Py) @nospecialize type @@ -281,15 +282,15 @@ end pyconvert_fix(::Type{T}, func) where {T} = x -> func(T, x) -const PYCONVERT_RULES_CACHE = Dict{Type, Dict{C.PyPtr, Vector{Function}}}() +const PYCONVERT_RULES_CACHE = Dict{Type,Dict{C.PyPtr,Vector{Function}}}() -@generated pyconvert_rules_cache(::Type{T}) where {T} = get!(Dict{C.PyPtr, Vector{Function}}, PYCONVERT_RULES_CACHE, T) +@generated pyconvert_rules_cache(::Type{T}) where {T} = get!(Dict{C.PyPtr,Vector{Function}}, PYCONVERT_RULES_CACHE, T) function pyconvert_rule_fast(::Type{T}, x::Py) where {T} if T isa Union - a = pyconvert_rule_fast(T.a, x) :: pyconvert_returntype(T.a) + a = pyconvert_rule_fast(T.a, x)::pyconvert_returntype(T.a) pyconvert_isunconverted(a) || return a - b = pyconvert_rule_fast(T.b, x) :: pyconvert_returntype(T.b) + b = pyconvert_rule_fast(T.b, x)::pyconvert_returntype(T.b) pyconvert_isunconverted(b) || return b elseif (T == Nothing) | (T == Missing) pyisnone(x) && return pyconvert_return(T()) @@ -318,7 +319,7 @@ function pytryconvert(::Type{T}, x_) where {T} # We can optimize the conversion for some types by overloading pytryconvert_fast. # It MUST give the same results as via the slower route using rules. - ans1 = pyconvert_rule_fast(T, x) :: pyconvert_returntype(T) + ans1 = pyconvert_rule_fast(T, x)::pyconvert_returntype(T) pyconvert_isunconverted(ans1) || return ans1 # get rules from the cache @@ -334,7 +335,7 @@ function pytryconvert(::Type{T}, x_) where {T} # apply the rules for rule in rules - ans2 = rule(x) :: pyconvert_returntype(T) + ans2 = rule(x)::pyconvert_returntype(T) pyconvert_isunconverted(ans2) || return ans2 end @@ -386,8 +387,8 @@ pyconvertarg(::Type{T}, x, name) where {T} = @autopy x @pyconvert T x_ begin end function init_pyconvert() - push!(PYCONVERT_EXTRATYPES, pyimport("io"=>"IOBase")) - push!(PYCONVERT_EXTRATYPES, pyimport("numbers"=>("Number", "Complex", "Real", "Rational", "Integral"))...) + push!(PYCONVERT_EXTRATYPES, pyimport("io" => "IOBase")) + push!(PYCONVERT_EXTRATYPES, pyimport("numbers" => ("Number", "Complex", "Real", "Rational", "Integral"))...) push!(PYCONVERT_EXTRATYPES, pyimport("collections.abc" => ("Iterable", "Sequence", "Set", "Mapping"))...) priority = PYCONVERT_PRIORITY_CANONICAL @@ -405,6 +406,7 @@ function init_pyconvert() pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority) pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date, priority) pyconvert_add_rule("datetime:time", Time, pyconvert_rule_time, priority) + pyconvert_add_rule("datetime:timedelta", Microsecond, pyconvert_rule_timedelta, priority) pyconvert_add_rule("builtins:BaseException", PyException, pyconvert_rule_exception, priority) priority = PYCONVERT_PRIORITY_NORMAL @@ -428,6 +430,9 @@ function init_pyconvert() pyconvert_add_rule("collections.abc:Sequence", Tuple, pyconvert_rule_iterable, priority) pyconvert_add_rule("collections.abc:Set", Set, pyconvert_rule_iterable, priority) pyconvert_add_rule("collections.abc:Mapping", Dict, pyconvert_rule_mapping, priority) + pyconvert_add_rule("datetime:timedelta", Millisecond, pyconvert_rule_timedelta, priority) + pyconvert_add_rule("datetime:timedelta", Second, pyconvert_rule_timedelta, priority) + pyconvert_add_rule("datetime:timedelta", Nanosecond, pyconvert_rule_timedelta, priority) priority = PYCONVERT_PRIORITY_FALLBACK pyconvert_add_rule("builtins:object", Py, pyconvert_rule_object, priority) diff --git a/src/Convert/rules.jl b/src/Convert/rules.jl index 8d0dc166..829a796a 100644 --- a/src/Convert/rules.jl +++ b/src/Convert/rules.jl @@ -133,7 +133,7 @@ function pyconvert_rule_range(::Type{R}, x::Py, ::Type{StepRange{T0,S0}}=Utils._ a′, c′ = promote(a, c - oftype(c, sign(b))) T2 = Utils._promote_type_bounded(T0, typeof(a′), typeof(c′), T1) S2 = Utils._promote_type_bounded(S0, typeof(c′), S1) - pyconvert_return(StepRange{T2, S2}(a′, b, c′)) + pyconvert_return(StepRange{T2,S2}(a′, b, c′)) end function pyconvert_rule_range(::Type{R}, x::Py, ::Type{UnitRange{T0}}=Utils._type_lb(R), ::Type{UnitRange{T1}}=Utils._type_ub(R)) where {R<:UnitRange,T0,T1} @@ -261,7 +261,7 @@ function pyconvert_rule_iterable(::Type{T}, xs::Py) where {T<:Tuple} zs = Any[] for x in xs if length(zs) < length(ts) - t = ts[length(zs) + 1] + t = ts[length(zs)+1] elseif isvararg t = vartype else @@ -282,7 +282,7 @@ for N in 0:16 n = pylen(xs) n == $N || return pyconvert_unconverted() $(( - :($z = @pyconvert($T, pytuple_getitem(xs, $(i-1)))) + :($z = @pyconvert($T, pytuple_getitem(xs, $(i - 1)))) for (i, T, z) in zip(1:N, Ts, zs) )...) return pyconvert_return(($(zs...),)) @@ -293,12 +293,12 @@ for N in 0:16 n = pylen(xs) n ≥ $N || return pyconvert_unconverted() $(( - :($z = @pyconvert($T, pytuple_getitem(xs, $(i-1)))) + :($z = @pyconvert($T, pytuple_getitem(xs, $(i - 1)))) for (i, T, z) in zip(1:N, Ts, zs) )...) vs = V[] - for i in $(N+1):n - v = @pyconvert(V, pytuple_getitem(xs, i-1)) + for i in $(N + 1):n + v = @pyconvert(V, pytuple_getitem(xs, i - 1)) push!(vs, v) end return pyconvert_return(($(zs...), vs...)) @@ -395,3 +395,47 @@ function pyconvert_rule_datetime(::Type{DateTime}, x::Py) iszero(mod(microseconds, 1000)) || return pyconvert_unconverted() return pyconvert_return(_base_datetime + Millisecond(div(microseconds, 1000) + 1000 * (seconds + 60 * 60 * 24 * days))) end + +function pyconvert_rule_timedelta(::Type{Nanosecond}, x::Py) + days = pyconvert(Int, x.days) + if abs(days) ≥ 106751 + # overflow + return pyconvert_unconverted() + end + seconds = pyconvert(Int, x.seconds) + microseconds = pyconvert(Int, x.microseconds) + return Nanosecond(((days * 3600 * 24 + seconds) * 1000000 + microseconds) * 1000) +end + +function pyconvert_rule_timedelta(::Type{Microsecond}, x::Py) + days = pyconvert(Int, x.days) + if abs(days) ≥ 106751990 + # overflow + return pyconvert_unconverted() + end + seconds = pyconvert(Int, x.seconds) + microseconds = pyconvert(Int, x.microseconds) + return Microsecond((days * 3600 * 24 + seconds) * 1000000 + microseconds) +end + +function pyconvert_rule_timedelta(::Type{Millisecond}, x::Py) + days = pyconvert(Int, x.days) + seconds = pyconvert(Int, x.seconds) + microseconds = pyconvert(Int, x.microseconds) + if mod(microseconds, 1000) != 0 + # inexact + return pyconvert_unconverted() + end + return Millisecond((days * 3600 * 24 + seconds) * 1000 + div(microseconds, 1000)) +end + +function pyconvert_rule_timedelta(::Type{Second}, x::Py) + days = pyconvert(Int, x.days) + seconds = pyconvert(Int, x.seconds) + microseconds = pyconvert(Int, x.microseconds) + if microseconds != 0 + # inexact + return pyconvert_unconverted() + end + return Second(days * 3600 * 24 + seconds) +end diff --git a/test/Convert.jl b/test/Convert.jl index 10e871d5..3d82c6af 100644 --- a/test/Convert.jl +++ b/test/Convert.jl @@ -223,6 +223,48 @@ end @test x1 === DateTime(2001, 2, 3, 4, 5, 6, 7) end +@testitem "timedelta → Nanosecond" begin + using Dates + td = pyimport("datetime").timedelta + @testset for x in [-1_000_000_000, -1_000_000, -1_000, -1, 0, 1, 1_000, 1_000_000, 1_000_000_000] + y = pyconvert(Nanosecond, td(microseconds=x)) + @test y === Nanosecond(x * 1000) + end + @test_throws Exception pyconvert(Nanosecond, td(days=200_000)) + @test_throws Exception pyconvert(Nanosecond, td(days=-200_000)) +end + +@testitem "timedelta → Microsecond" begin + using Dates + td = pyimport("datetime").timedelta + @testset for x in [-1_000_000_000, -1_000_000, -1_000, -1, 0, 1, 1_000, 1_000_000, 1_000_000_000] + y = pyconvert(Microsecond, td(microseconds=x)) + @test y === Microsecond(x) + end + @test_throws Exception pyconvert(Microsecond, td(days=200_000_000)) + @test_throws Exception pyconvert(Microsecond, td(days=-200_000_000)) +end + +@testitem "timedelta → Millisecond" begin + using Dates + td = pyimport("datetime").timedelta + @testset for x in [-1_000_000_000, -1_000_000, -1_000, -1, 0, 1, 1_000, 1_000_000, 1_000_000_000] + y = pyconvert(Millisecond, td(microseconds=x*1000)) + @test y === Millisecond(x) + end + @test_throws Exception pyconvert(Millisecond, td(microseconds=1)) +end + +@testitem "timedelta → Second" begin + using Dates + td = pyimport("datetime").timedelta + @testset for x in [-1_000_000_000, -1_000_000, -1_000, -1, 0, 1, 1_000, 1_000_000, 1_000_000_000] + y = pyconvert(Second, td(seconds=x)) + @test y === Second(x) + end + @test_throws Exception pyconvert(Second, td(microseconds=1000)) +end + @testitem "pyconvert_add_rule (#364)" begin id = string(rand(UInt128), base=16) pyexec("""