diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 0599cc0b6ec36..449b08580ae1a 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -521,7 +521,9 @@ struct DataTypeLayout size::UInt32 nfields::UInt32 npointers::UInt32 + nhidden_pointers::UInt32 firstptr::Int32 + firsthiddenptr::Int32 alignment::UInt16 flags::UInt16 # haspadding : 1; diff --git a/src/datatype.c b/src/datatype.c index 9132fa4f208bb..f27d0eceac321 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -80,6 +80,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu tn->partial = NULL; tn->atomicfields = NULL; tn->constfields = NULL; + tn->hiddenptrfields = NULL; tn->max_methods = 0; tn->constprop_heustic = 0; return tn; @@ -126,6 +127,7 @@ static uint32_t _hash_djb2(uint32_t hash, const char *mem, size_t s) JL_NOTSAFEP return hash; } + static uint32_t _hash_layout_djb2(uintptr_t _layout, void *unused) JL_NOTSAFEPOINT { (void)unused; @@ -138,11 +140,20 @@ static uint32_t _hash_layout_djb2(uintptr_t _layout, void *unused) JL_NOTSAFEPOI const char *pointers = jl_dt_layout_ptrs(layout); assert(pointers); size_t pointers_size = layout->first_ptr < 0 ? 0 : (layout->npointers << layout->flags.fielddesc_type); + const char *hidden_pointers = NULL; + size_t hidden_ptrs_size = 0; + if (layout->first_hidden_ptr >= 0) { + hidden_pointers = jl_dt_layout_hidden_ptrs(layout); + hidden_ptrs_size = layout->nhidden_pointers << layout->flags.fielddesc_type; + } uint_t hash = 5381; hash = _hash_djb2(hash, (char *)layout, own_size); hash = _hash_djb2(hash, fields, fields_size); hash = _hash_djb2(hash, pointers, pointers_size); + if (hidden_ptrs_size > 0) { + hash = _hash_djb2(hash, hidden_pointers, hidden_ptrs_size); + } return hash; } @@ -163,6 +174,13 @@ static int layout_eq(void *_l1, void *_l2, void *unused) JL_NOTSAFEPOINT size_t pointers_size = l1->first_ptr < 0 ? 0 : (l1->npointers << l1->flags.fielddesc_type); if (memcmp(p1, p2, pointers_size)) return 0; + if (l1->first_hidden_ptr >= 0 && l2->first_hidden_ptr >= 0) { + const char *h1 = jl_dt_layout_hidden_ptrs(l1); + const char *h2 = jl_dt_layout_hidden_ptrs(l2); + size_t hidden_ptrs_size = l1->nhidden_pointers << l1->flags.fielddesc_type; + if (memcmp(h1, h2, hidden_ptrs_size)) + return 0; + } return 1; } @@ -174,15 +192,18 @@ HTIMPL_R(layoutcache, _hash_layout_djb2, layout_eq) static htable_t layoutcache; static int layoutcache_initialized = 0; + static jl_datatype_layout_t *jl_get_layout(uint32_t sz, uint32_t nfields, uint32_t npointers, + uint32_t nhidden_pointers, uint32_t alignment, int haspadding, int isbitsegal, int arrayelem, jl_fielddesc32_t desc[], - uint32_t pointers[]) JL_NOTSAFEPOINT + uint32_t pointers[], + uint32_t hidden_pointers[]) JL_NOTSAFEPOINT { assert(alignment); // should have been verified by caller @@ -212,11 +233,13 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, } } int32_t first_ptr = (npointers > 0 ? (int32_t)pointers[0] : -1); + int32_t first_hidden_ptr = (nhidden_pointers > 0 ? (int32_t)hidden_pointers[0] : -1); // allocate a new descriptor, on the stack if possible. size_t fields_size = nfields * jl_fielddesc_size(fielddesc_type); size_t pointers_size = first_ptr < 0 ? 0 : (npointers << fielddesc_type); - size_t flddesc_sz = sizeof(jl_datatype_layout_t) + fields_size + pointers_size; + size_t hidden_ptrs_size = first_hidden_ptr < 0 ? 0 : (nhidden_pointers << fielddesc_type); + size_t flddesc_sz = sizeof(jl_datatype_layout_t) + fields_size + pointers_size + hidden_ptrs_size; int should_malloc = flddesc_sz >= jl_page_size; jl_datatype_layout_t *mallocmem = (jl_datatype_layout_t *)(should_malloc ? malloc(flddesc_sz) : NULL); jl_datatype_layout_t *allocamem = (jl_datatype_layout_t *)(should_malloc ? NULL : alloca(flddesc_sz)); @@ -233,6 +256,8 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, flddesc->flags.padding = 0; flddesc->npointers = npointers; flddesc->first_ptr = first_ptr; + flddesc->nhidden_pointers = nhidden_pointers; + flddesc->first_hidden_ptr = first_hidden_ptr; // fill out the fields of the new descriptor jl_fielddesc8_t *desc8 = (jl_fielddesc8_t *)jl_dt_layout_fields(flddesc); @@ -272,6 +297,25 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, } } + // fill out hidden pointer descriptors + if (first_hidden_ptr >= 0 && hidden_pointers) { + uint8_t *hptrs8 = (uint8_t *)jl_dt_layout_hidden_ptrs(flddesc); + uint16_t *hptrs16 = (uint16_t *)jl_dt_layout_hidden_ptrs(flddesc); + uint32_t *hptrs32 = (uint32_t *)jl_dt_layout_hidden_ptrs(flddesc); + uint32_t *src_descs = (uint32_t *)hidden_pointers; + for (size_t i = 0; i < nhidden_pointers; i++) { + if (fielddesc_type == 0) { + hptrs8[i] = src_descs[i]; + } + else if (fielddesc_type == 1) { + hptrs16[i] = src_descs[i]; + } + else { + hptrs32[i] = src_descs[i]; + } + } + } + if (__unlikely(!layoutcache_initialized)) { htable_new(&layoutcache, 4096); layoutcache_initialized = 1; @@ -299,6 +343,7 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, return ret; } + // Determine if homogeneous tuple with fields of type t will have // a special alignment and vector-ABI beyond normal rules for aggregates. // Return special alignment if one exists, 0 if normal alignment rules hold. @@ -502,7 +547,7 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) jl_value_t *addrspace = jl_tparam2(st); if (!jl_is_typevar(eltype) && !jl_is_type(eltype)) { // this is expected to have a layout, but since it is not constructable, we don't care too much what it is - static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), {0}}; + static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), {0}}; st->layout = &opaque_ptr_layout; st->has_concrete_subtype = 0; return; @@ -571,7 +616,7 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) else arrayelem = 0; assert(!st->layout); - st->layout = jl_get_layout(elsz, nfields, npointers, al, haspadding, isbitsegal, arrayelem, NULL, pointers); + st->layout = jl_get_layout(elsz, nfields, npointers, 0, al, haspadding, isbitsegal, arrayelem, NULL, pointers, NULL); st->zeroinit = zi; //st->has_concrete_subtype = 1; //st->isbitstype = 0; @@ -630,17 +675,17 @@ void jl_compute_field_offsets(jl_datatype_t *st) // if we have no fields, we can trivially skip the rest if (st == jl_symbol_type || st == jl_string_type) { // opaque layout - heap-allocated blob - static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, 0, -1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; st->layout = &opaque_byte_layout; return; } else if (st == jl_simplevector_type || st == jl_module_type) { - static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; st->layout = &opaque_ptr_layout; return; } else { - static const jl_datatype_layout_t singleton_layout = {0, 0, 0, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t singleton_layout = {0, 0, 0, 0, -1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; st->layout = &singleton_layout; } } @@ -672,6 +717,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) size_t descsz = nfields * sizeof(jl_fielddesc32_t); jl_fielddesc32_t *desc; uint32_t *pointers; + uint32_t *hidden_pointers; int should_malloc = descsz >= jl_page_size; if (should_malloc) desc = (jl_fielddesc32_t*)malloc_s(descsz); @@ -685,12 +731,22 @@ void jl_compute_field_offsets(jl_datatype_t *st) int homogeneous = 1; int needlock = 0; uint32_t npointers = 0; + uint32_t nhidden_pointers = 0; jl_value_t *firstty = jl_field_type(st, 0); for (i = 0; i < nfields; i++) { jl_value_t *fld = jl_field_type(st, i); int isatomic = jl_field_isatomic(st, i); + int ishiddenptr = jl_field_ishiddenptr(st, i); size_t fsz = 0, al = 1; - if (jl_islayout_inline(fld, &fsz, &al) && (!isatomic || jl_is_datatype(fld))) { // aka jl_datatype_isinlinealloc + if (ishiddenptr) { + fsz = sizeof(void*); + al = fsz; + if (al > MAX_ALIGN) + al = MAX_ALIGN; + desc[i].isptr = 0; // Hidden pointers are stored as non-pointer fields + nhidden_pointers++; + } + else if (jl_islayout_inline(fld, &fsz, &al) && (!isatomic || jl_is_datatype(fld))) { // aka jl_datatype_isinlinealloc if (__unlikely(fsz > max_size)) // Should never happen throw_ovf(should_malloc, desc, st, fsz); @@ -724,6 +780,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) if (!zeroinit) zeroinit = ((jl_datatype_t*)fld)->zeroinit; npointers += fld_npointers; + nhidden_pointers += ((jl_datatype_t*)fld)->layout->nhidden_pointers; } } else { @@ -782,25 +839,61 @@ void jl_compute_field_offsets(jl_datatype_t *st) pointers = (uint32_t*)malloc_s(npointers * sizeof(uint32_t)); else pointers = (uint32_t*)alloca(npointers * sizeof(uint32_t)); + if (should_malloc && nhidden_pointers) { + hidden_pointers = (uint32_t*)malloc_s(nhidden_pointers * sizeof(uint32_t)); + } else { + hidden_pointers = (uint32_t*)alloca(nhidden_pointers * sizeof(uint32_t)); + } size_t ptr_i = 0; + size_t hptr_i = 0; for (i = 0; i < nfields; i++) { jl_value_t *fld = jl_field_type(st, i); uint32_t offset = desc[i].offset / sizeof(jl_value_t**); - if (desc[i].isptr) + int ishiddenptr = jl_field_ishiddenptr(st, i); + if (ishiddenptr) { + // Direct hidden pointer field + hidden_pointers[hptr_i] = offset; + hptr_i++; + } + else if (desc[i].isptr) pointers[ptr_i++] = offset; else if (jl_is_datatype(fld)) { + // Handle nested datatype with regular/hidden pointers int j, npointers = ((jl_datatype_t*)fld)->layout->npointers; for (j = 0; j < npointers; j++) { pointers[ptr_i++] = offset + jl_ptr_offset((jl_datatype_t*)fld, j); } + // Copy hidden pointers from nested field + int nhidden = ((jl_datatype_t*)fld)->layout->nhidden_pointers; + for (j = 0; j < nhidden; j++) { + hidden_pointers[hptr_i] = offset + jl_hidden_ptr_offset((jl_datatype_t*)fld, j); + hptr_i++; + } } } assert(ptr_i == npointers); - st->layout = jl_get_layout(sz, nfields, npointers, alignm, haspadding, isbitsegal, 0, desc, pointers); + assert(hptr_i == nhidden_pointers); + st->layout = jl_get_layout(sz, nfields, npointers, nhidden_pointers, alignm, haspadding, isbitsegal, 0, desc, pointers, hidden_pointers); + + // Validation: Ensure no overlap between pointer and hidden pointer offsets + if (npointers > 0 && nhidden_pointers > 0) { + for (size_t p = 0; p < npointers; p++) { + for (size_t hp = 0; hp < nhidden_pointers; hp++) { + if (pointers[p] == hidden_pointers[hp]) { + jl_errorf("Field offset conflict: field at offset %u appears in both regular pointer offsets and hidden pointer offsets", + pointers[p]); + } + } + } + } + + // All hidden pointers are assumed to be PtrOrOffset type if (should_malloc) { free(desc); if (npointers) free(pointers); + if (nhidden_pointers) + free(hidden_pointers); } st->zeroinit = zeroinit; } @@ -813,7 +906,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) return; } -JL_DLLEXPORT jl_datatype_t *jl_new_datatype( +JL_DLLEXPORT jl_datatype_t *jl_new_datatype_with_hiddenptrs( jl_sym_t *name, jl_module_t *module, jl_datatype_t *super, @@ -822,7 +915,8 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( jl_svec_t *ftypes, jl_svec_t *fattrs, int abstract, int mutabl, - int ninitialized) + int ninitialized, + uint32_t* hiddenptrfields) { jl_datatype_t *t = NULL; jl_typename_t *tn = NULL; @@ -912,6 +1006,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( } tn->atomicfields = atomicfields; tn->constfields = constfields; + tn->hiddenptrfields = hiddenptrfields; if (t->name->wrapper == NULL) { t->name->wrapper = (jl_value_t*)t; @@ -933,6 +1028,22 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( return t; } +JL_DLLEXPORT jl_datatype_t *jl_new_datatype( + jl_sym_t *name, + jl_module_t *module, + jl_datatype_t *super, + jl_svec_t *parameters, + jl_svec_t *fnames, + jl_svec_t *ftypes, + jl_svec_t *fattrs, + int abstract, int mutabl, + int ninitialized) +{ + return jl_new_datatype_with_hiddenptrs( + name, module, super, parameters, fnames, ftypes, fattrs, + abstract, mutabl, ninitialized, NULL); +} + JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *module, jl_datatype_t *super, jl_svec_t *parameters, size_t nbits) @@ -962,7 +1073,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t * bt->ismutationfree = 1; bt->isidentityfree = 1; bt->isbitstype = (parameters == jl_emptysvec); - bt->layout = jl_get_layout(nbytes, 0, 0, alignm, 0, 1, 0, NULL, NULL); + bt->layout = jl_get_layout(nbytes, 0, 0, 0, alignm, 0, 1, 0, NULL, NULL, NULL); bt->instance = NULL; return bt; } diff --git a/src/jltypes.c b/src/jltypes.c index cda47118541dd..734290cd37756 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3007,27 +3007,29 @@ void jl_init_types(void) JL_GC_DISABLED jl_typename_type->name->mt = jl_nonfunction_mt; jl_typename_type->super = jl_any_type; jl_typename_type->parameters = jl_emptysvec; - jl_typename_type->name->n_uninitialized = 16 - 2; - jl_typename_type->name->names = jl_perm_symsvec(16, "name", "module", + jl_typename_type->name->n_uninitialized = 17 - 2; + jl_typename_type->name->names = jl_perm_symsvec(17, "name", "module", "names", "atomicfields", "constfields", "wrapper", "Typeofwrapper", "cache", "linearcache", "mt", "partial", "hash", "n_uninitialized", "flags", // "abstract", "mutable", "mayinlinealloc", - "max_methods", "constprop_heuristic"); + "max_methods", "constprop_heuristic", + "hiddenptrfields"); const static uint32_t typename_constfields[1] = { 0x00003a27 }; // (1<<0)|(1<<1)|(1<<2)|(1<<5)|(1<<9)|(1<<11)|(1<<12)|(1<<13) ; TODO: put back (1<<3)|(1<<4) in this list const static uint32_t typename_atomicfields[1] = { 0x00000180 }; // (1<<7)|(1<<8) jl_typename_type->name->constfields = typename_constfields; jl_typename_type->name->atomicfields = typename_atomicfields; jl_precompute_memoized_dt(jl_typename_type, 1); - jl_typename_type->types = jl_svec(16, jl_symbol_type, jl_any_type /*jl_module_type*/, + jl_typename_type->types = jl_svec(17, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_simplevector_type, jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/, jl_type_type, jl_type_type, jl_simplevector_type, jl_simplevector_type, jl_methtable_type, jl_any_type, jl_any_type /*jl_long_type*/, jl_any_type /*jl_int32_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, - jl_any_type /*jl_uint8_type*/); + jl_any_type /*jl_uint8_type*/, + jl_any_type/*jl_voidpointer_type*/); jl_methtable_type->name = jl_new_typename_in(jl_symbol("MethodTable"), core, 0, 1); jl_methtable_type->name->wrapper = (jl_value_t*)jl_methtable_type; @@ -3338,11 +3340,12 @@ void jl_init_types(void) JL_GC_DISABLED memory_datatype->ismutationfree = 0; jl_datatype_t *jl_memoryref_supertype = (jl_datatype_t*)jl_apply_type1((jl_value_t*)jl_ref_type, jl_svecref(tv, 1)); + const static uint32_t memoryref_hiddenptrfields[1] = { 0x00000001 }; // (1<<0) - field 0 is a hidden pointer jl_datatype_t *memoryref_datatype = - jl_new_datatype(jl_symbol("GenericMemoryRef"), core, jl_memoryref_supertype, tv, + jl_new_datatype_with_hiddenptrs(jl_symbol("GenericMemoryRef"), core, jl_memoryref_supertype, tv, jl_perm_symsvec(2, "ptr_or_offset", "mem"), jl_svec(2, pointer_void, memory_datatype), - jl_emptysvec, 0, 0, 2); + jl_emptysvec, 0, 0, 2, memoryref_hiddenptrfields); jl_genericmemoryref_typename = memoryref_datatype->name; jl_genericmemoryref_type = (jl_unionall_t*)jl_genericmemoryref_typename->wrapper; memoryref_datatype->ismutationfree = 0; @@ -3856,6 +3859,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_typename_type->types, 13, jl_uint8_type); jl_svecset(jl_typename_type->types, 14, jl_uint8_type); jl_svecset(jl_typename_type->types, 15, jl_uint8_type); + jl_svecset(jl_typename_type->types, 16, jl_voidpointer_type); // hiddenptrfields jl_svecset(jl_methtable_type->types, 4, jl_long_type); jl_svecset(jl_methtable_type->types, 5, jl_module_type); jl_svecset(jl_methtable_type->types, 6, jl_array_any_type); diff --git a/src/julia.h b/src/julia.h index c3b90dac09c11..f664655cb9131 100644 --- a/src/julia.h +++ b/src/julia.h @@ -188,9 +188,12 @@ JL_EXTENSION typedef struct _jl_genericmemory_t { #endif } jl_genericmemory_t; +// Special type annotation for fields containing hidden pointers +typedef void * jl_hidden_ptr_ptr_or_offset_t; + JL_EXTENSION typedef struct { JL_DATA_TYPE - void *ptr_or_offset; + jl_hidden_ptr_ptr_or_offset_t ptr_or_offset; jl_genericmemory_t *mem; } jl_genericmemoryref_t; @@ -546,6 +549,7 @@ typedef struct { uint8_t _reserved:5; uint8_t max_methods; // override for inference's max_methods setting (0 = no additional limit or relaxation) uint8_t constprop_heustic; // override for inference's constprop heuristic + const uint32_t *hiddenptrfields; // if any fields are hidden pointers, we record them here } jl_typename_t; typedef struct { @@ -576,8 +580,10 @@ typedef struct { typedef struct { uint32_t size; uint32_t nfields; - uint32_t npointers; // number of pointers embedded inside - int32_t first_ptr; // index of the first pointer (or -1) + uint32_t npointers; // number of direct object pointers + uint32_t nhidden_pointers; // number of hidden/internal pointers + int32_t first_ptr; // index of the first direct pointer (or -1) + int32_t first_hidden_ptr; // index of the first hidden pointer (or -1) uint16_t alignment; // strictest alignment over all fields struct { // combine these fields into a struct so that we can take addressof them uint16_t haspadding : 1; // has internal undefined bytes @@ -600,6 +606,11 @@ typedef struct { // uint16_t ptr16[npointers]; // uint32_t ptr32[npointers]; // }; + // union { // offsets relative to data start in words + // uint8_t ptr8[nhidden_pointers]; + // uint16_t ptr16[nhidden_pointers]; + // uint32_t ptr32[nhidden_pointers]; + // }; } jl_datatype_layout_t; typedef struct _jl_datatype_t { @@ -1714,6 +1725,34 @@ static inline uint32_t jl_ptr_offset(jl_datatype_t *st, int i) JL_NOTSAFEPOINT } } +// Access functions for hidden pointer descriptors +static inline const char *jl_dt_layout_hidden_ptrs(const jl_datatype_layout_t *l) JL_NOTSAFEPOINT +{ + const char *ptrs = jl_dt_layout_ptrs(l); + size_t direct_ptrs_size = l->first_ptr < 0 ? 0 : (l->npointers << l->flags.fielddesc_type); + return ptrs + direct_ptrs_size; +} + +static inline uint32_t jl_hidden_ptr_offset(jl_datatype_t *st, int i) JL_NOTSAFEPOINT +{ + const jl_datatype_layout_t *ly = st->layout; + assert(i >= 0 && (size_t)i < ly->nhidden_pointers); + const void *hidden_ptrs = jl_dt_layout_hidden_ptrs(ly); + + if (ly->flags.fielddesc_type == 0) { + return ((const uint8_t*)hidden_ptrs)[i]; + } + else if (ly->flags.fielddesc_type == 1) { + return ((const uint16_t*)hidden_ptrs)[i]; + } + else { + assert(ly->flags.fielddesc_type == 2); + return ((const uint32_t*)hidden_ptrs)[i]; + } +} + + + static inline int jl_field_isatomic(jl_datatype_t *st, int i) JL_NOTSAFEPOINT { const uint32_t *atomicfields = st->name->atomicfields; @@ -1737,6 +1776,17 @@ static inline int jl_field_isconst(jl_datatype_t *st, int i) JL_NOTSAFEPOINT return 0; } +static inline int jl_field_ishiddenptr(jl_datatype_t *st, int i) JL_NOTSAFEPOINT +{ + const uint32_t *hiddenptrfields = st->name->hiddenptrfields; + if (hiddenptrfields != NULL) { + if (hiddenptrfields[i / 32] & (1 << (i % 32))) + return 1; + } + return 0; +} + + // basic predicates ----------------------------------------------------------- #define jl_is_nothing(v) (((jl_value_t*)(v)) == ((jl_value_t*)jl_nothing)) @@ -2039,6 +2089,16 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_svec_t *fattrs, int abstract, int mutabl, int ninitialized); +JL_DLLEXPORT jl_datatype_t *jl_new_datatype_with_hiddenptrs(jl_sym_t *name, + jl_module_t *module, + jl_datatype_t *super, + jl_svec_t *parameters, + jl_svec_t *fnames, + jl_svec_t *ftypes, + jl_svec_t *fattrs, + int abstract, int mutabl, + int ninitialized, + uint32_t* hiddenptrfields); JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *module, jl_datatype_t *super, diff --git a/src/staticdata.c b/src/staticdata.c index 6c737539feb8c..139205d28ad13 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1949,6 +1949,8 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize; if (!is_foreign_type && dt->layout->first_ptr != -1) fldsize += np << dt->layout->flags.fielddesc_type; + if (!is_foreign_type && dt->layout->first_hidden_ptr != -1) + fldsize += dt->layout->nhidden_pointers << dt->layout->flags.fielddesc_type; uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*)); write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream newdt->layout = NULL; // relocation offset