Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
generalize insert-backedges to insert in any world
Browse files Browse the repository at this point in the history
Rather than a binary valid/not-valid, we track the exact later world
that deleted it, relative to when we first assumed it may be valid.
vtjnash committed Feb 25, 2023

Unverified

This user has not yet uploaded their public signing key.
1 parent f0eadd0 commit 43dabee
Showing 2 changed files with 133 additions and 123 deletions.
6 changes: 4 additions & 2 deletions src/staticdata.c
Original file line number Diff line number Diff line change
@@ -2243,6 +2243,7 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new
}

if (edges) {
size_t world = jl_atomic_load_acquire(&jl_world_counter);
jl_collect_missing_backedges(jl_type_type_mt);
jl_collect_missing_backedges(jl_nonfunction_mt);
// jl_collect_extext_methods_from_mod and jl_collect_missing_backedges also accumulate data in callers_with_edges.
@@ -2252,7 +2253,7 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new
*method_roots_list = jl_alloc_vec_any(0);
// Collect the new method roots
jl_collect_new_roots(*method_roots_list, *new_specializations, worklist_key);
jl_collect_edges(*edges, *ext_targets, *new_specializations);
jl_collect_edges(*edges, *ext_targets, *new_specializations, world);
}
assert(edges_map == NULL); // jl_collect_edges clears this when done

@@ -3270,7 +3271,8 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im
// Add roots to methods
jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored));
// Handle edges
jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets, (jl_array_t*)new_specializations); // restore external backedges (needs to be last)
size_t world = jl_atomic_load_acquire(&jl_world_counter);
jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)ext_targets, (jl_array_t*)new_specializations, world); // restore external backedges (needs to be last)
// reinit ccallables
jl_reinit_ccallable(&ccallable_list, base, NULL);
arraylist_free(&ccallable_list);
250 changes: 129 additions & 121 deletions src/staticdata_utils.c
Original file line number Diff line number Diff line change
@@ -418,9 +418,8 @@ static void jl_record_edges(jl_method_instance_t *caller, arraylist_t *wq, jl_ar
// Extract `edges` and `ext_targets` from `edges_map`
// `edges` = [caller1, targets_indexes1, ...], the list of methods and their edges
// `ext_targets` is [invokesig1, callee1, matches1, ...], the edges for each target
static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *external_cis)
static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *external_cis, size_t world)
{
size_t world = jl_atomic_load_acquire(&jl_world_counter);
htable_t external_mis;
htable_new(&external_mis, 0);
if (external_cis) {
@@ -821,38 +820,40 @@ static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key)
}
}

static jl_value_t *ulong JL_ALWAYS_LEAFTYPE = NULL;

// verify that these edges intersect with the same methods as before
static jl_array_t *jl_verify_edges(jl_array_t *targets)
static jl_array_t *jl_verify_edges(jl_array_t *targets, size_t minworld)
{
size_t world = jl_atomic_load_acquire(&jl_world_counter);
size_t i, l = jl_array_len(targets) / 3;
jl_array_t *valids = jl_alloc_array_1d(jl_array_uint8_type, l);
memset(jl_array_data(valids), 1, l);
if (ulong == NULL)
ulong = jl_apply_array_type((jl_value_t*)jl_ulong_type, 1);
jl_array_t *maxvalids = jl_alloc_array_1d(ulong, l);
memset(jl_array_data(maxvalids), 0, l * sizeof(size_t));
jl_value_t *loctag = NULL;
jl_value_t *matches = NULL;
JL_GC_PUSH3(&valids, &matches, &loctag);
JL_GC_PUSH3(&maxvalids, &matches, &loctag);
for (i = 0; i < l; i++) {
jl_value_t *invokesig = jl_array_ptr_ref(targets, i * 3);
jl_value_t *callee = jl_array_ptr_ref(targets, i * 3 + 1);
jl_value_t *expected = jl_array_ptr_ref(targets, i * 3 + 2);
int valid = 1;
size_t min_valid = 0;
size_t max_valid = ~(size_t)0;
if (invokesig) {
assert(callee && "unsupported edge");
jl_methtable_t *mt = jl_method_get_table(((jl_method_instance_t*)callee)->def.method);
if ((jl_value_t*)mt == jl_nothing) {
valid = 0;
max_valid = 0;
}
else {
matches = jl_gf_invoke_lookup_worlds(invokesig, (jl_value_t*)mt, world, &min_valid, &max_valid);
matches = jl_gf_invoke_lookup_worlds(invokesig, (jl_value_t*)mt, minworld, &min_valid, &max_valid);
if (matches == jl_nothing) {
valid = 0;
max_valid = 0;
}
else {
matches = (jl_value_t*)((jl_method_match_t*)matches)->method;
if (matches != expected) {
valid = 0;
max_valid = 0;
}
}
}
@@ -867,15 +868,15 @@ static jl_array_t *jl_verify_edges(jl_array_t *targets)
int ambig = 0;
// TODO: possibly need to included ambiguities too (for the optimizer correctness)?
matches = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing,
-1, 0, world, &min_valid, &max_valid, &ambig);
-1, 0, minworld, &min_valid, &max_valid, &ambig);
if (matches == jl_nothing) {
valid = 0;
max_valid = 0;
}
else {
// setdiff!(matches, expected)
size_t j, k, ins = 0;
if (jl_array_len(matches) != jl_array_len(expected)) {
valid = 0;
max_valid = 0;
}
for (k = 0; k < jl_array_len(matches); k++) {
jl_method_t *match = ((jl_method_match_t*)jl_array_ptr_ref(matches, k))->method;
@@ -887,18 +888,18 @@ static jl_array_t *jl_verify_edges(jl_array_t *targets)
// intersection has a new method or a method was
// deleted--this is now probably no good, just invalidate
// everything about it now
valid = 0;
max_valid = 0;
if (!_jl_debug_method_invalidation)
break;
jl_array_ptr_set(matches, ins++, match);
}
}
if (!valid && _jl_debug_method_invalidation)
if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation)
jl_array_del_end((jl_array_t*)matches, jl_array_len(matches) - ins);
}
}
jl_array_uint8_set(valids, i, valid);
if (!valid && _jl_debug_method_invalidation) {
((size_t*)(jl_array_data(maxvalids)))[i] = max_valid;
if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) {
jl_array_ptr_1d_push(_jl_debug_method_invalidation, invokesig ? (jl_value_t*)invokesig : callee);
loctag = jl_cstr_to_string("insert_backedges_callee");
jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag);
@@ -911,161 +912,168 @@ static jl_array_t *jl_verify_edges(jl_array_t *targets)
//ios_puts(valid ? "valid\n" : "INVALID\n", ios_stderr);
}
JL_GC_POP();
return valids;
return maxvalids;
}

// Combine all edges relevant to a method into the visited table
static void jl_verify_methods(jl_array_t *edges, jl_array_t *valids, htable_t *visited)
// Combine all edges relevant to a method to initialize the maxvalids list
static jl_array_t *jl_verify_methods(jl_array_t *edges, jl_array_t *maxvalids)
{
jl_value_t *loctag = NULL;
JL_GC_PUSH1(&loctag);
jl_array_t *maxvalids2 = NULL;
JL_GC_PUSH2(&loctag, &maxvalids2);
size_t i, l = jl_array_len(edges) / 2;
htable_new(visited, l);
maxvalids2 = jl_alloc_array_1d(jl_typeof(maxvalids), l);
size_t *maxvalids2_data = jl_array_data(maxvalids2);
memset(maxvalids2_data, 0, l * sizeof(size_t));
for (i = 0; i < l; i++) {
jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i);
assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method));
jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1);
assert(jl_typeis((jl_value_t*)callee_ids, jl_array_int32_type));
int valid = 1;
if (callee_ids == NULL) {
// serializing the edges had failed
valid = 0;
maxvalids2_data[i] = 0;
}
else {
int32_t *idxs = (int32_t*)jl_array_data(callee_ids);
size_t j;
for (j = 0; valid && j < idxs[0]; j++) {
maxvalids2_data[i] = ~(size_t)0;
for (j = 0; j < idxs[0]; j++) {
int32_t idx = idxs[j + 1];
valid = jl_array_uint8_ref(valids, idx);
if (!valid && _jl_debug_method_invalidation) {
size_t max_valid = ((size_t*)(jl_array_data(maxvalids)))[idx];
if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) {
jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)caller);
loctag = jl_cstr_to_string("verify_methods");
jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag);
loctag = jl_box_int32((int32_t)idx);
jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag);
}
if (max_valid < maxvalids2_data[i])
maxvalids2_data[i] = max_valid;
if (max_valid == 0)
break;
}
}
ptrhash_put(visited, caller, (void*)(((char*)HT_NOTFOUND) + valid + 1));
//jl_static_show((JL_STREAM*)ios_stderr, (jl_value_t*)caller);
//ios_puts(valid ? "valid\n" : "INVALID\n", ios_stderr);
// HT_NOTFOUND: valid (no invalid edges)
// HT_NOTFOUND + 1: invalid
// HT_NOTFOUND + 2: need to scan
// HT_NOTFOUND + 3 + depth: in-progress
//ios_puts(maxvalid2_data[i] == ~(size_t)0 ? "valid\n" : "INVALID\n", ios_stderr);
}
JL_GC_POP();
return maxvalids2;
}


// Visit the entire call graph, starting from edges[idx] to determine if that method is valid
// Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable
static int jl_verify_graph_edge(jl_array_t *edges, int idx, htable_t *visited, arraylist_t *stack)
// and slightly modified with an early termination option once the computation reaches its minimum
static int jl_verify_graph_edge(size_t *maxvalids2_data, jl_array_t *edges, size_t idx, arraylist_t *visited, arraylist_t *stack)
{
jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, idx * 2);
assert(jl_is_method_instance(caller) && jl_is_method(caller->def.method));
int found = (char*)ptrhash_get(visited, (void*)caller) - (char*)HT_NOTFOUND;
if (found == 0)
return 1; // NOTFOUND == valid
if (found == 1)
return 0; // invalid
if (found != 2)
return found - 1; // depth
found = 0;
if (maxvalids2_data[idx] == 0) {
visited->items[idx] = (void*)1;
return 0;
}
size_t cycle = (size_t)visited->items[idx];
if (cycle != 0)
return cycle - 1; // depth remaining
jl_value_t *cause = NULL;
arraylist_push(stack, (void*)caller);
int depth = stack->len;
ptrhash_put(visited, (void*)caller, (void*)((char*)HT_NOTFOUND + 3 + depth)); // change 2 to in-progress at depth
arraylist_push(stack, (void*)idx);
size_t depth = stack->len;
visited->items[idx] = (void*)(1 + depth);
jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, idx * 2 + 1);
assert(jl_typeis((jl_value_t*)callee_ids, jl_array_int32_type));
int32_t *idxs = (int32_t*)jl_array_data(callee_ids);
int cycle = 0;
size_t i, n = jl_array_len(callee_ids);
for (i = idxs[0] + 1; i < n; i++) {
int32_t idx = idxs[i];
int child_found = jl_verify_graph_edge(edges, idx, visited, stack);
if (child_found == 0) {
int32_t childidx = idxs[i];
int child_cycle = jl_verify_graph_edge(maxvalids2_data, edges, childidx, visited, stack);
size_t child_max_valid = maxvalids2_data[childidx];
if (child_max_valid < maxvalids2_data[idx]) {
maxvalids2_data[idx] = child_max_valid;
cause = jl_array_ptr_ref(edges, childidx * 2);
}
if (child_max_valid == 0) {
// found what we were looking for, so terminate early
found = 1;
cause = jl_array_ptr_ref(edges, idx * 2);
break;
}
else if (child_found >= 2 && child_found - 2 < cycle) {
else if (child_cycle && child_cycle < cycle) {
// record the cycle will resolve at depth "cycle"
cycle = child_found - 2;
assert(cycle);
cycle = child_cycle;
}
}
if (!found && cycle && cycle != depth)
return cycle + 2;
size_t max_valid = maxvalids2_data[idx];
if (max_valid != 0 && cycle && cycle != depth)
return cycle;
// If we are the top of the current cycle, now mark all other parts of
// our cycle with what we found.
// Or if we found a backedge, also mark all of the other parts of the
// cycle as also having an backedge.
// Or if we found a failed edge, also mark all of the other parts of the
// cycle as also having an failed edge.
while (stack->len >= depth) {
void *mi = arraylist_pop(stack);
assert((char*)ptrhash_get(visited, mi) - (char*)HT_NOTFOUND == 4 + stack->len);
if (found)
ptrhash_put(visited, mi, (void*)((char*)HT_NOTFOUND + 1 + found));
else
ptrhash_remove(visited, mi); // assign as NOTFOUND in table
if (_jl_debug_method_invalidation && found) {
jl_value_t *loctag = NULL;
JL_GC_PUSH1(&loctag);
jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)mi);
loctag = jl_cstr_to_string("verify_methods");
jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag);
jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)cause);
JL_GC_POP();
size_t childidx = (size_t)arraylist_pop(stack);
assert(visited->items[childidx] == (void*)(2 + stack->len));
if (idx != childidx) {
if (max_valid < maxvalids2_data[childidx])
maxvalids2_data[childidx] = max_valid;
if (_jl_debug_method_invalidation && max_valid != ~(size_t)0) {
jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(edges, childidx * 2);
jl_value_t *loctag = NULL;
JL_GC_PUSH1(&loctag);
jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)mi);
loctag = jl_cstr_to_string("verify_methods");
jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag);
jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)cause);
JL_GC_POP();
}
}
visited->items[childidx] = (void*)1;
}
return found ? 0 : 1;
return 0;
}

// Visit all entries in edges, verify if they are valid
static jl_array_t *jl_verify_graph(jl_array_t *edges, htable_t *visited)
static void jl_verify_graph(jl_array_t *edges, jl_array_t *maxvalids2)
{
arraylist_t stack;
arraylist_t stack, visited;
arraylist_new(&stack, 0);
size_t i, n = jl_array_len(edges) / 2;
jl_array_t *valids = jl_alloc_array_1d(jl_array_uint8_type, n);
JL_GC_PUSH1(&valids);
int8_t *valids_data = (int8_t*)jl_array_data(valids);
for (i = 0; i < n; i++)
valids_data[i] = jl_verify_graph_edge(edges, i, visited, &stack);
arraylist_new(&visited, n);
memset(visited.items, 0, n * sizeof(size_t));
size_t *maxvalids2_data = (size_t*)jl_array_data(maxvalids2);
for (i = 0; i < n; i++) {
assert(visited.items[i] == (void*)0 || visited.items[i] == (void*)1);
int child_cycle = jl_verify_graph_edge(maxvalids2_data, edges, i, &visited, &stack);
assert(child_cycle == 0); (void)child_cycle;
assert(stack.len == 0);
assert(visited.items[i] == (void*)1);
}
arraylist_free(&stack);
JL_GC_POP();
return valids;
arraylist_free(&visited);
}

// Restore backedges to external targets
// `edges` = [caller1, targets_indexes1, ...], the list of worklist-owned methods calling external methods.
// `ext_targets` is [invokesig1, callee1, matches1, ...], the global set of non-worklist callees of worklist-owned methods.
static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *ci_list)
static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_array_t *ci_list, size_t minworld)
{
// determine which CodeInstance objects are still valid in our image
size_t world = jl_atomic_load_acquire(&jl_world_counter);
jl_array_t *valids = jl_verify_edges(ext_targets);
jl_array_t *valids = jl_verify_edges(ext_targets, minworld);
JL_GC_PUSH1(&valids);
htable_t visited;
htable_new(&visited, 0);
jl_verify_methods(edges, valids, &visited); // consumes valids, creates visited
valids = jl_verify_graph(edges, &visited); // consumes visited, creates valids
valids = jl_verify_methods(edges, valids); // consumes edges valids, initializes methods valids
jl_verify_graph(edges, valids); // propagates methods valids for each edge
size_t i, l;

// next build a map from external MethodInstances to their CodeInstance for insertion
l = jl_array_len(ci_list);
htable_reset(&visited, l);
htable_t visited;
htable_new(&visited, l);
for (i = 0; i < l; i++) {
jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(ci_list, i);
assert(ci->min_world == world);
assert(ci->min_world == minworld);
if (ci->max_world == 1) { // sentinel value: has edges to external callables
ptrhash_put(&visited, (void*)ci->def, (void*)ci);
}
else {
assert(ci->max_world == ~(size_t)0);
jl_method_instance_t *caller = ci->def;
if (ci->inferred && jl_rettype_inferred(caller, world, ~(size_t)0) == jl_nothing) {
if (ci->inferred && jl_rettype_inferred(caller, minworld, ~(size_t)0) == jl_nothing) {
jl_mi_cache_insert(caller, ci);
}
//jl_static_show((jl_stream*)ios_stderr, (jl_value_t*)caller);
@@ -1077,28 +1085,28 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a
l = jl_array_len(edges) / 2;
for (i = 0; i < l; i++) {
jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i);
int valid = jl_array_uint8_ref(valids, i);
if (!valid)
continue;
// if this callee is still valid, add all the backedges
jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1);
int32_t *idxs = (int32_t*)jl_array_data(callee_ids);
for (size_t j = 0; j < idxs[0]; j++) {
int32_t idx = idxs[j + 1];
jl_value_t *invokesig = jl_array_ptr_ref(ext_targets, idx * 3);
jl_value_t *callee = jl_array_ptr_ref(ext_targets, idx * 3 + 1);
if (callee && jl_is_method_instance(callee)) {
jl_method_instance_add_backedge((jl_method_instance_t*)callee, invokesig, caller);
}
else {
jl_value_t *sig = callee == NULL ? invokesig : callee;
jl_methtable_t *mt = jl_method_table_for(sig);
// FIXME: rarely, `callee` has an unexpected `Union` signature,
// see https://github.com/JuliaLang/julia/pull/43990#issuecomment-1030329344
// Fix the issue and turn this back into an `assert((jl_value_t*)mt != jl_nothing)`
// This workaround exposes us to (rare) 265-violations.
if ((jl_value_t*)mt != jl_nothing)
jl_method_table_add_backedge(mt, sig, (jl_value_t*)caller);
size_t maxvalid = ((size_t*)(jl_array_data(valids)))[i];
if (maxvalid == ~(size_t)0) {
// if this callee is still valid, add all the backedges
jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1);
int32_t *idxs = (int32_t*)jl_array_data(callee_ids);
for (size_t j = 0; j < idxs[0]; j++) {
int32_t idx = idxs[j + 1];
jl_value_t *invokesig = jl_array_ptr_ref(ext_targets, idx * 3);
jl_value_t *callee = jl_array_ptr_ref(ext_targets, idx * 3 + 1);
if (callee && jl_is_method_instance(callee)) {
jl_method_instance_add_backedge((jl_method_instance_t*)callee, invokesig, caller);
}
else {
jl_value_t *sig = callee == NULL ? invokesig : callee;
jl_methtable_t *mt = jl_method_table_for(sig);
// FIXME: rarely, `callee` has an unexpected `Union` signature,
// see https://github.com/JuliaLang/julia/pull/43990#issuecomment-1030329344
// Fix the issue and turn this back into an `assert((jl_value_t*)mt != jl_nothing)`
// This workaround exposes us to (rare) 265-violations.
if ((jl_value_t*)mt != jl_nothing)
jl_method_table_add_backedge(mt, sig, (jl_value_t*)caller);
}
}
}
// then enable any methods associated with it
@@ -1108,9 +1116,9 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a
// have some new external code to use
assert(jl_is_code_instance(ci));
jl_code_instance_t *codeinst = (jl_code_instance_t*)ci;
assert(codeinst->min_world == world && codeinst->inferred);
codeinst->max_world = ~(size_t)0;
if (jl_rettype_inferred(caller, world, ~(size_t)0) == jl_nothing) {
assert(codeinst->min_world == minworld && codeinst->inferred);
codeinst->max_world = maxvalid;
if (jl_rettype_inferred(caller, minworld, maxvalid) == jl_nothing) {
jl_mi_cache_insert(caller, codeinst);
}
}

0 comments on commit 43dabee

Please sign in to comment.