Skip to content

Commit

Permalink
layout: support pointers inlining into parents [NFCI] (#33886)
Browse files Browse the repository at this point in the history 
Includes codegen support for immutable objects that contain pointers
appearing on stack (well, in registers, since LLVM support of
non-integral addrspace pointers inside aggregates in memory is poor),
and includes layout support, so that most (non-self-referential)
immutable objects can be stored inline inside their parent allocation.

Currently fully disabled, aside from some optimizations and improvements
to object_id / isa tests.

Co-Authored-By: Oscar Blumberg <oscar.blumberg@ens.fr>
  • Loading branch information
@vtjnash @carnaval @JeffBezanson
2 people authored and JeffBezanson committed Dec 17, 2019
1 parent fdcaa06 commit 630a551
Show file tree
Hide file tree
Showing 25 changed files with 1,072 additions and 412 deletions.
79 changes: 44 additions & 35 deletions base/array.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ size(a::Array{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(

asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...)

allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_array_store_unboxed, Cint, (Any,), T) != Cint(0))
allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0))

"""
Base.isbitsunion(::Type{T})
Expand All @@ -177,14 +177,20 @@ false
isbitsunion(u::Union) = allocatedinline(u)
isbitsunion(x) = false

function _unsetindex!(A::Array{T}, i::Int) where {T}
@inbounds function _unsetindex!(A::Array{T}, i::Int) where {T}
@boundscheck checkbounds(A, i)
t = @_gc_preserve_begin A
p = Ptr{Ptr{Cvoid}}(pointer(A, i))
if !allocatedinline(T)
t = @_gc_preserve_begin A
p = Ptr{Ptr{Cvoid}}(pointer(A))
unsafe_store!(p, C_NULL, i)
@_gc_preserve_end t
unsafe_store!(p, C_NULL)
elseif T isa DataType
if !datatype_pointerfree(T)
for j = 1:(Core.sizeof(T) ÷ Core.sizeof(Ptr{Cvoid}))
unsafe_store!(p, C_NULL, j)
end
end
end
@_gc_preserve_end t
return A
end

Expand Down Expand Up @@ -255,19 +261,41 @@ the same manner as C.
function unsafe_copyto!(dest::Array{T}, doffs, src::Array{T}, soffs, n) where T
t1 = @_gc_preserve_begin dest
t2 = @_gc_preserve_begin src
if isbitsunion(T)
destp = pointer(dest, doffs)
srcp = pointer(src, soffs)
if !allocatedinline(T)
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
dest, destp, src, srcp, n)
elseif isbitstype(T)
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
pointer(dest, doffs), pointer(src, soffs), n * aligned_sizeof(T))
destp, srcp, n * aligned_sizeof(T))
elseif isbitsunion(T)
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
destp, srcp, n * aligned_sizeof(T))
# copy selector bytes
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), dest) + doffs - 1,
ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), src) + soffs - 1,
n)
elseif allocatedinline(T)
unsafe_copyto!(pointer(dest, doffs), pointer(src, soffs), n)
else
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
dest, pointer(dest, doffs), src, pointer(src, soffs), n)
# handle base-case: everything else above was just optimizations
@inbounds if destp < srcp || destp > srcp + n
for i = 1:n
if isassigned(src, soffs + i - 1)
dest[doffs + i - 1] = src[soffs + i - 1]
else
_unsetindex!(dest, doffs + i - 1)
end
end
else
for i = n:-1:1
if isassigned(src, soffs + i - 1)
dest[doffs + i - 1] = src[soffs + i - 1]
else
_unsetindex!(dest, doffs + i - 1)
end
end
end
end
@_gc_preserve_end t2
@_gc_preserve_end t1
Expand Down Expand Up @@ -1566,32 +1594,13 @@ function vcat(arrays::Vector{T}...) where T
n += length(a)
end
arr = Vector{T}(undef, n)
ptr = pointer(arr)
if isbitsunion(T)
selptr = ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), arr)
end
elsz = aligned_sizeof(T)
t = @_gc_preserve_begin arr
nd = 1
for a in arrays
na = length(a)
nba = na * elsz
if isbitsunion(T)
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ptr, a, nba)
# copy selector bytes
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
selptr, ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), a), na)
selptr += na
elseif allocatedinline(T)
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ptr, a, nba)
else
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
arr, ptr, a, pointer(a), na)
end
ptr += nba
@assert nd + na <= 1 + length(arr) # Concurrent modification of arrays?
unsafe_copyto!(arr, nd, a, 1, na)
nd += na
end
@_gc_preserve_end t
return arr
end

Expand Down
1 change: 1 addition & 0 deletions base/reflection.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ datatype_fieldtypes(x::DataType) = ccall(:jl_get_fieldtypes, Any, (Any,), x)
struct DataTypeLayout
nfields::UInt32
npointers::UInt32
firstptr::Int32
alignment::UInt32
# alignment : 9;
# haspadding : 1;
Expand Down
2 changes: 1 addition & 1 deletion base/refpointer.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ RefArray(x::AbstractArray{T}, i::Int=1, roots::Nothing=nothing) where {T} = RefA
convert(::Type{Ref{T}}, x::AbstractArray{T}) where {T} = RefArray(x, 1)

function unsafe_convert(P::Type{Ptr{T}}, b::RefArray{T}) where T
if datatype_pointerfree(RefValue{T})
if allocatedinline(T)
p = pointer(b.x, b.i)
elseif isconcretetype(T) && T.mutable
p = pointer_from_objref(b.x[b.i])
Expand Down
2 changes: 1 addition & 1 deletion base/refvalue.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ RefValue(x::T) where {T} = RefValue{T}(x)
isassigned(x::RefValue) = isdefined(x, :x)

function unsafe_convert(P::Type{Ptr{T}}, b::RefValue{T}) where T
if datatype_pointerfree(RefValue{T})
if allocatedinline(T)
p = pointer_from_objref(b)
elseif isconcretetype(T) && T.mutable
p = pointer_from_objref(b.x)
Expand Down
70 changes: 47 additions & 23 deletions src/array.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,6 @@ char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT
return ((char*)jl_array_data(a)) + ((jl_array_ndims(a) == 1 ? (a->maxsize - a->offset) : jl_array_len(a)) * a->elsize) + a->offset;
}

JL_DLLEXPORT int jl_array_store_unboxed(jl_value_t *eltype) JL_NOTSAFEPOINT
{
size_t fsz = 0, al = 0;
return jl_islayout_inline(eltype, &fsz, &al);
}

STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT
{
if (a->flags.how == 3) {
Expand All @@ -53,7 +47,7 @@ size_t jl_arr_xtralloc_limit = 0;
#define MAXINTVAL (((size_t)-1)>>1)

static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
int isunboxed, int isunion, int elsz)
int isunboxed, int hasptr, int isunion, int elsz)
{
jl_ptls_t ptls = jl_get_ptls_states();
size_t i, tot, nel=1;
Expand Down Expand Up @@ -101,7 +95,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
// No allocation or safepoint allowed after this
a->flags.how = 0;
data = (char*)a + doffs;
if ((tot > 0 && !isunboxed) || isunion)
if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
memset(data, 0, tot);
}
else {
Expand All @@ -113,7 +107,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
// No allocation or safepoint allowed after this
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
if (!isunboxed || isunion)
if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
// need to zero out isbits union array selector bytes to ensure a valid type index
memset(data, 0, tot);
}
Expand All @@ -127,6 +121,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
#endif
a->flags.ndims = ndims;
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = hasptr;
a->elsize = elsz;
a->flags.isshared = 0;
a->flags.isaligned = 1;
Expand All @@ -135,9 +130,12 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
a->nrows = nel;
a->maxsize = nel;
}
else if (a->flags.ndims != ndims) {
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
}
else {
size_t *adims = &a->nrows;
for(i=0; i < ndims; i++)
for (i = 0; i < ndims; i++)
adims[i] = dims[i];
}

Expand All @@ -152,6 +150,7 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
jl_type_error_rt("Array", "element type", (jl_value_t*)jl_type_type, eltype);
int isunboxed = jl_islayout_inline(eltype, &elsz, &al);
int isunion = jl_is_uniontype(eltype);
int hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
if (!isunboxed) {
elsz = sizeof(void*);
al = elsz;
Expand All @@ -160,13 +159,13 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
elsz = LLT_ALIGN(elsz, al);
}

return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
}

jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims,
int isunboxed, int isunion, int elsz)
int isunboxed, int hasptr, int isunion, int elsz)
{
return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
}

#ifndef JL_NDEBUG
Expand Down Expand Up @@ -224,10 +223,12 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
"reinterpret from alignment %d bytes to alignment %d bytes not allowed",
(int) oldalign, (int) align);
a->flags.ptrarray = 0;
a->flags.hasptr = data->flags.hasptr;
}
else {
a->elsize = sizeof(void*);
a->flags.ptrarray = 1;
a->flags.hasptr = 0;
}

// if data is itself a shared wrapper,
Expand All @@ -247,6 +248,9 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
a->nrows = l;
a->maxsize = l;
}
else if (a->flags.ndims != ndims) {
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
}
else {
size_t *adims = &a->nrows;
size_t l = 1;
Expand Down Expand Up @@ -281,6 +285,7 @@ JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
a->flags.isaligned = 0;
a->elsize = 1;
a->flags.ptrarray = 0;
a->flags.hasptr = 0;
jl_array_data_owner(a) = str;
a->flags.how = 3;
a->flags.isshared = 1;
Expand All @@ -300,12 +305,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
jl_array_t *a;
jl_value_t *eltype = jl_tparam0(atype);

int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
int isunboxed = jl_stored_inline(eltype);
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
size_t elsz;
unsigned align;
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
Expand All @@ -328,6 +333,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
#endif
a->elsize = LLT_ALIGN(elsz, align);
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
a->flags.ndims = 1;
a->flags.isshared = 1;
a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers
Expand Down Expand Up @@ -366,12 +372,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
return jl_ptr_to_array_1d(atype, data, nel, own_buffer);
jl_value_t *eltype = jl_tparam0(atype);

int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
int isunboxed = jl_stored_inline(eltype);
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
size_t elsz;
unsigned align;
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
Expand All @@ -394,6 +400,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
#endif
a->elsize = LLT_ALIGN(elsz, align);
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
a->flags.ndims = ndims;
a->offset = 0;
a->flags.isshared = 1;
Expand All @@ -408,6 +415,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
}

assert(ndims != 1); // handled above
if (a->flags.ndims != ndims)
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
memcpy(&a->nrows, dims, ndims * sizeof(size_t));
return a;
}
Expand Down Expand Up @@ -559,8 +568,16 @@ JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i)

JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i)
{
if (a->flags.ptrarray)
if (a->flags.ptrarray) {
return ((jl_value_t**)jl_array_data(a))[i] != NULL;
}
else if (a->flags.hasptr) {
jl_datatype_t *eltype = (jl_datatype_t*)jl_tparam0(jl_typeof(a));
assert(eltype->layout->first_ptr >= 0);
jl_value_t **slot =
(jl_value_t**)(&((char*)a->data)[i*a->elsize] + eltype->layout->first_ptr);
return *slot != NULL;
}
return 1;
}

Expand All @@ -585,6 +602,8 @@ JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *rhs
return;
}
jl_assign_bits(&((char*)a->data)[i * a->elsize], rhs);
if (a->flags.hasptr)
jl_gc_multi_wb(jl_array_owner(a), rhs);
}
else {
((jl_value_t**)a->data)[i] = rhs;
Expand All @@ -598,6 +617,11 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i)
jl_bounds_error_int((jl_value_t*)a, i + 1);
if (a->flags.ptrarray)
((jl_value_t**)a->data)[i] = NULL;
else if (a->flags.hasptr) {
size_t elsize = a->elsize;
jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0);
memset(&((jl_value_t**)a->data)[i], 0, elsize);
}
}

// at this size and bigger, allocate resized array data with malloc directly
Expand Down Expand Up @@ -809,7 +833,7 @@ STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc,
#endif
a->nrows = newnrows;
a->data = newdata;
if (a->flags.ptrarray) {
if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
memset(newdata + idx * elsz, 0, nbinc);
}
else if (isbitsunion) {
Expand Down Expand Up @@ -890,7 +914,7 @@ STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx,
a->length = newnrows;
#endif
a->nrows = newnrows;
if (a->flags.ptrarray) {
if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
memset(data + idx * elsz, 0, inc * elsz);
}
}
Expand Down Expand Up @@ -1129,7 +1153,7 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary)
int isunion = jl_is_uniontype(jl_tparam0(jl_typeof(ary)));
jl_array_t *new_ary = _new_array_(jl_typeof(ary), jl_array_ndims(ary),
&ary->nrows, !ary->flags.ptrarray,
isunion, elsz);
ary->flags.hasptr, isunion, elsz);
memcpy(new_ary->data, ary->data, len * elsz);
// ensure isbits union arrays copy their selector bytes correctly
if (jl_array_isbitsunion(ary))
Expand Down
Loading

0 comments on commit 630a551

Please sign in to comment.