Skip to content

Commit

Permalink
Merge pull request #25261 from JuliaLang/kf/iterate
Browse files Browse the repository at this point in the history 
Iteration protocol change
  • Loading branch information
@Keno
Keno authored May 18, 2018
2 parents c2030ed + 62fbad2 commit aa301aa
Show file tree
Hide file tree
Showing 105 changed files with 1,310 additions and 2,006 deletions.
104 changes: 59 additions & 45 deletions base/abstractarray.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,9 +284,9 @@ julia> first([1; 2; 3; 4])
```
"""
function first(itr)
state = start(itr)
done(itr, state) && throw(ArgumentError("collection must be non-empty"))
next(itr, state)[1]
x = iterate(itr)
x === nothing && throw(ArgumentError("collection must be non-empty"))
x[1]
end

"""
Expand Down Expand Up @@ -635,10 +635,12 @@ emptymutable(itr, ::Type{U}) where {U} = Vector{U}()

function copyto!(dest::AbstractArray, src)
destiter = eachindex(dest)
state = start(destiter)
y = iterate(destiter)
for x in src
i, state = next(destiter, state)
dest[i] = x
y === nothing &&
throw(ArgumentError(string("source has fewer elements than required")))
dest[y[1]] = x
y = iterate(destiter, y[2])
end
return dest
end
Expand All @@ -657,25 +659,24 @@ function copyto!(dest::AbstractArray, dstart::Integer, src, sstart::Integer)
if (sstart < 1)
throw(ArgumentError(string("source start offset (",sstart,") is < 1")))
end
st = start(src)
y = iterate(src)
for j = 1:(sstart-1)
if done(src, st)
if y === nothing
throw(ArgumentError(string("source has fewer elements than required, ",
"expected at least ",sstart,", got ",j-1)))
end
_, st = next(src, st)
y = iterate(src, y[2])
end
dn = done(src, st)
if dn
if y === nothing
throw(ArgumentError(string("source has fewer elements than required, ",
"expected at least ",sstart,", got ",sstart-1)))
end
i = Int(dstart)
while !dn
val, st = next(src, st)
while y != nothing
val, st = y
dest[i] = val
i += 1
dn = done(src, st)
y = iterate(src, st)
end
return dest
end
Expand All @@ -690,18 +691,19 @@ function copyto!(dest::AbstractArray, dstart::Integer, src, sstart::Integer, n::
sstart < 1 && throw(ArgumentError(string("source start offset (",sstart,") is < 1")))
throw(BoundsError(dest, dstart:dmax))
end
st = start(src)
y = iterate(src)
for j = 1:(sstart-1)
if done(src, st)
if y === nothing
throw(ArgumentError(string("source has fewer elements than required, ",
"expected at least ",sstart,", got ",j-1)))
end
_, st = next(src, st)
y = iterate(src, y[2])
end
i = Int(dstart)
while i <= dmax && !done(src, st)
val, st = next(src, st)
while i <= dmax && y !== nothing
val, st = y
@inbounds dest[i] = val
y = iterate(src, st)
i += 1
end
i <= dmax && throw(BoundsError(dest, i))
Expand Down Expand Up @@ -823,9 +825,11 @@ zero(x::AbstractArray{T}) where {T} = fill!(similar(x), zero(T))
# While the definitions for IndexLinear are all simple enough to inline on their
# own, IndexCartesian's CartesianIndices is more complicated and requires explicit
# inlining.
start(A::AbstractArray) = (@_inline_meta; itr = eachindex(A); (itr, start(itr)))
next(A::AbstractArray, i) = (@_propagate_inbounds_meta; (idx, s) = next(i[1], i[2]); (A[idx], (i[1], s)))
done(A::AbstractArray, i) = (@_propagate_inbounds_meta; done(i[1], i[2]))
function iterate(A::AbstractArray, state=(eachindex(A),))
y = iterate(state...)
y === nothing && return nothing
A[y[1]], (state[1], tail(y)...)
end

isempty(a::AbstractArray) = (_length(a) == 0)

Expand Down Expand Up @@ -2042,12 +2046,17 @@ function hash(a::AbstractArray{T}, h::UInt) where T
h += hashaa_seed
h += hash(size(a))

state = start(a)
done(a, state) && return h
x1, state = next(a, state)
done(a, state) && return hash(x1, h)
x2, state = next(a, state)
done(a, state) && return hash(x2, hash(x1, h))
y1 = iterate(a)
y1 === nothing && return h
y2 = iterate(a, y1[2])
y2 === nothing && return hash(y1[1], h)
y = iterate(a, y2[2])
y === nothing && return hash(y2[1], hash(y1[1], h))
x1, x2 = y1[1], y2[1]

# For the rest of the function, we keep three elements worth of state,
# x1, x2, y[1], with `y` potentially being `nothing` if there's only
# two elements remaining

# Check whether the array is equal to a range, and hash the elements
# at the beginning of the array as such as long as they match this assumption
Expand All @@ -2056,6 +2065,10 @@ function hash(a::AbstractArray{T}, h::UInt) where T
if isa(a, AbstractVector) && applicable(-, x2, x1)
n = 1
local step, laststep, laststate

h = hash(x1, h)
h += hashr_seed

while true
# If overflow happens with entries of the same type, a cannot be equal
# to a range with more than two elements because more extreme values
Expand All @@ -2077,47 +2090,48 @@ function hash(a::AbstractArray{T}, h::UInt) where T
end
n > 1 && !isequal(step, laststep) && break
n += 1
x1 = x2
laststep = step
laststate = state
done(a, state) && break
x2, state = next(a, state)
if y === nothing
# The array matches a range exactly
return hash(x2, hash(n, h))
end
x1, x2 = x2, y[1]
y = iterate(a, y[2])
end

h = hash(first(a), h)
h += hashr_seed
# Always hash at least the two first elements as a range (even in case of overflow)
if n < 2
h = hash(2, h)
h = hash(x2, h)
done(a, state) && return h
x1 = x2
x2, state = next(a, state)
h = hash(y2[1], h)
@assert y !== nothing
x1, x2 = x2, y[1]
y = iterate(a, y[2])
else
h = hash(n, h)
h = hash(x1, h)
done(a, laststate) && return h
end
end

# Hash elements which do not correspond to a range (if any)
# Hash elements which do not correspond to a range
while true
if isequal(x2, x1)
# For repeated elements, use run length encoding
# This allows efficient hashing of sparse arrays
runlength = 2
while !done(a, state)
x2, state = next(a, state)
while y !== nothing
# No need to update x1 (it's isequal x2)
x2 = y[1]
y = iterate(a, y[2])
isequal(x1, x2) || break
runlength += 1
end
h += hashrle_seed
h = hash(runlength, h)
end
h = hash(x1, h)
done(a, state) && break
x1 = x2
x2, state = next(a, state)
y === nothing && break
x1, x2 = x2, y[1]
y = iterate(a, y[2])
end
!isequal(x2, x1) && (h = hash(x2, h))
return h
Expand Down
31 changes: 13 additions & 18 deletions base/abstractdict.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,10 @@ isempty(v::Union{KeySet,ValueIterator}) = isempty(v.dict)
_tt2(::Type{Pair{A,B}}) where {A,B} = B
eltype(::Type{ValueIterator{D}}) where {D} = _tt2(eltype(D))

start(v::Union{KeySet,ValueIterator}) = start(v.dict)
done(v::Union{KeySet,ValueIterator}, state) = done(v.dict, state)

function next(v::KeySet, state)
n = next(v.dict, state)
n[1][1], n[2]
end

function next(v::ValueIterator, state)
n = next(v.dict, state)
n[1][2], n[2]
function iterate(v::Union{KeySet,ValueIterator}, state...)
y = iterate(v.dict, state...)
y === nothing && return nothing
return (y[1][isa(v, KeySet) ? 1 : 2], y[2])
end

in(k, v::KeySet) = get(v.dict, k, secret_table_token) !== secret_table_token
Expand Down Expand Up @@ -670,9 +663,11 @@ end

_oidd_nextind(a, i) = reinterpret(Int, ccall(:jl_eqtable_nextind, Csize_t, (Any, Csize_t), a, i))

start(d::IdDict) = _oidd_nextind(d.ht, 0)
done(d::IdDict, i) = (i == -1)
next(d::IdDict{K, V}, i) where {K, V} = (Pair{K, V}(d.ht[i + 1]::K, d.ht[i + 2]::V), _oidd_nextind(d.ht, i + 2))
function iterate(d::IdDict{K,V}, idx=0) where {K, V}
idx = _oidd_nextind(d.ht, idx)
idx == -1 && return nothing
return (Pair{K, V}(d.ht[idx + 1]::K, d.ht[idx + 2]::V), idx + 2)
end

length(d::IdDict) = d.count

Expand Down Expand Up @@ -711,9 +706,9 @@ empty!(s::IdSet) = (empty!(s.dict); s)

filter!(f, d::IdSet) = unsafe_filter!(f, d)

start(s::IdSet) = start(s.dict)
done(s::IdSet, state) = done(s.dict, state)
function next(s::IdSet, state)
((k, _), i) = next(s.dict, state)
function iterate(s::IdSet, state...)
y = iterate(s.dict, state...)
y === nothing && return nothing
((k, _), i) = y
return (k, i)
end
Loading

2 comments on commit aa301aa

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Executing the daily benchmark build, I will reply here when finished:

@nanosoldier runbenchmarks(ALL, isdaily = true)

@nanosoldier
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your benchmark job has completed - possible performance regressions were detected. A full report can be found here. cc @ararslan

Please sign in to comment.