From 585a6c158d3b8781cd37ad03d38dffb33deec3e9 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 4 Nov 2024 21:33:10 +0000 Subject: [PATCH] further defer jl_insert_backedges after loading Finish fully breaking the dependency between method insertions and inferring whether the cache is valid. The cache should be inferable in parallel and in aggregate after all loading is finished. This prepares us for moving this code into Julia (Core.Compiler) next. --- src/staticdata.c | 5 +- src/staticdata_utils.c | 177 +++++++++++++++++++++++------------------ 2 files changed, 102 insertions(+), 80 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index 0d609db03aebc7..5172ef34f89c98 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -4040,12 +4040,13 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i // allocate a world for the new methods, and insert them there, invalidating content as needed size_t world = jl_atomic_load_relaxed(&jl_world_counter) + 1; jl_activate_methods(extext_methods, internal_methods, world); + // TODO: inject new_ext_cis into caches here, so the system can see them immediately as potential condidates (before validation) // allow users to start running in this updated world jl_atomic_store_release(&jl_world_counter, world); - // but one of those immediate users is going to be our cache updates - jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)new_ext_cis, world); // restore external backedges (needs to be last) // now permit more methods to be added again JL_UNLOCK(&world_counter_lock); + // but one of those immediate users is going to be our cache insertions + jl_insert_backedges((jl_array_t*)edges, (jl_array_t*)new_ext_cis); // restore existing caches (needs to be last) // reinit ccallables jl_reinit_ccallable(&ccallable_list, base, pkgimage_handle); arraylist_free(&ccallable_list); diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index b69c1edb5429b9..f12e10f8491615 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -751,56 +751,58 @@ static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) } } -static size_t verify_invokesig(jl_value_t *invokesig, jl_method_t *expected, size_t minworld) +static void verify_invokesig(jl_value_t *invokesig, jl_method_t *expected, size_t world, size_t *minworld, size_t *maxworld) { assert(jl_is_type(invokesig)); assert(jl_is_method(expected)); - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; if (jl_egal(invokesig, expected->sig)) { // the invoke match is `expected` for `expected->sig`, unless `expected` is invalid - if (jl_atomic_load_relaxed(&expected->deleted_world) < max_valid) - max_valid = 0; + *minworld = jl_atomic_load_relaxed(&expected->primary_world); + *maxworld = jl_atomic_load_relaxed(&expected->deleted_world); + assert(*minworld <= world); + if (*maxworld < world) + *maxworld = 0; } else { + *minworld = 1; + *maxworld = ~(size_t)0; jl_methtable_t *mt = jl_method_get_table(expected); if ((jl_value_t*)mt == jl_nothing) { - max_valid = 0; + *maxworld = 0; } else { - jl_value_t *matches = jl_gf_invoke_lookup_worlds(invokesig, (jl_value_t*)mt, minworld, &min_valid, &max_valid); + jl_value_t *matches = jl_gf_invoke_lookup_worlds(invokesig, (jl_value_t*)mt, world, minworld, maxworld); if (matches == jl_nothing) { - max_valid = 0; + *maxworld = 0; } else { if (((jl_method_match_t*)matches)->method != expected) { - max_valid = 0; + *maxworld = 0; } } } } - return max_valid; } -static size_t verify_call(jl_value_t *sig, jl_svec_t *expecteds, size_t i, size_t n, size_t minworld, jl_value_t **matches JL_REQUIRE_ROOTED_SLOT) +static void verify_call(jl_value_t *sig, jl_svec_t *expecteds, size_t i, size_t n, size_t world, size_t *minworld, size_t *maxworld, jl_value_t **matches JL_REQUIRE_ROOTED_SLOT) { // verify that these edges intersect with the same methods as before - size_t min_valid = 0; - size_t max_valid = ~(size_t)0; + *minworld = 1; + *maxworld = ~(size_t)0; int ambig = 0; // TODO: possibly need to included ambiguities too (for the optimizer correctness)? jl_value_t *result = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, _jl_debug_method_invalidation ? INT32_MAX : n, - 0, minworld, &min_valid, &max_valid, &ambig); + 0, world, minworld, maxworld, &ambig); *matches = result; if (result == jl_nothing) { - max_valid = 0; + *maxworld = 0; } else { // setdiff!(result, expected) size_t j, k, ins = 0; if (jl_array_nrows(result) != n) { - max_valid = 0; + *maxworld = 0; } for (k = 0; k < jl_array_nrows(result); k++) { jl_method_t *match = ((jl_method_match_t*)jl_array_ptr_ref(result, k))->method; @@ -822,29 +824,33 @@ static size_t verify_call(jl_value_t *sig, jl_svec_t *expecteds, size_t i, size_ // intersection has a new method or a method was // deleted--this is now probably no good, just invalidate // everything about it now - max_valid = 0; + *maxworld = 0; if (!_jl_debug_method_invalidation) break; jl_array_ptr_set(result, ins++, match); } } - if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) + if (*maxworld != ~(size_t)0 && _jl_debug_method_invalidation) jl_array_del_end((jl_array_t*)result, jl_array_nrows(result) - ins); } - return max_valid; } // Test all edges relevant to a method: //// Visit the entire call graph, starting from edges[idx] to determine if that method is valid //// Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable //// and slightly modified with an early termination option once the computation reaches its minimum -static int jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, size_t *maxworld, arraylist_t *stack, htable_t *visiting) +static int jl_verify_method(jl_code_instance_t *codeinst, size_t *minworld, size_t *maxworld, arraylist_t *stack, htable_t *visiting) { + size_t world = jl_atomic_load_relaxed(&codeinst->min_world); size_t max_valid2 = jl_atomic_load_relaxed(&codeinst->max_world); if (max_valid2 != WORLD_AGE_REVALIDATION_SENTINEL) { + *minworld = world; *maxworld = max_valid2; return 0; } + *minworld = 1; + size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); + *maxworld = current_world; assert(jl_is_method_instance(codeinst->def) && jl_is_method(codeinst->def->def.method)); void **bp = ptrhash_bp(visiting, codeinst); if (*bp != HT_NOTFOUND) @@ -862,6 +868,7 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, size_ // verify current edges for (size_t j = 0; j < jl_svec_len(callees); ) { jl_value_t *edge = jl_svecref(callees, j); + size_t min_valid2; size_t max_valid2; assert(!jl_is_method(edge)); // `Method`-edge isn't allowed for the optimized one-edge format if (jl_is_code_instance(edge)) @@ -869,14 +876,14 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, size_ if (jl_is_method_instance(edge)) { jl_method_instance_t *mi = (jl_method_instance_t*)edge; sig = jl_type_intersection(mi->def.method->sig, (jl_value_t*)mi->specTypes); // TODO: ?? - max_valid2 = verify_call(sig, callees, j, 1, minworld, &matches); + verify_call(sig, callees, j, 1, world, &min_valid2, &max_valid2, &matches); sig = NULL; j += 1; } else if (jl_is_long(edge)) { jl_value_t *sig = jl_svecref(callees, j + 1); size_t nedges = jl_unbox_long(edge); - max_valid2 = verify_call(sig, callees, j + 2, nedges, minworld, &matches); + verify_call(sig, callees, j + 2, nedges, world, &min_valid2, &max_valid2, &matches); j += 2 + nedges; edge = sig; } @@ -896,9 +903,11 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, size_ assert(jl_is_method(callee)); meth = (jl_method_t*)callee; } - max_valid2 = verify_invokesig(edge, meth, minworld); + verify_invokesig(edge, meth, world, &min_valid2, &max_valid2); j += 2; } + if (*minworld < min_valid2) + *minworld = min_valid2; if (*maxworld > max_valid2) *maxworld = max_valid2; if (max_valid2 != ~(size_t)0 && _jl_debug_method_invalidation) { @@ -917,14 +926,19 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, size_ // verify recursive edges (if valid, or debugging) size_t cycle = depth; jl_code_instance_t *cause = codeinst; - if (*maxworld == ~(size_t)0 || _jl_debug_method_invalidation) { + if (*maxworld != 0 || _jl_debug_method_invalidation) { for (size_t j = 0; j < jl_svec_len(callees); j++) { jl_value_t *edge = jl_svecref(callees, j); if (!jl_is_code_instance(edge)) continue; jl_code_instance_t *callee = (jl_code_instance_t*)edge; - size_t max_valid2 = ~(size_t)0; - size_t child_cycle = jl_verify_method(callee, minworld, &max_valid2, stack, visiting); + size_t min_valid2; + size_t max_valid2; + size_t child_cycle = jl_verify_method(callee, &min_valid2, &max_valid2, stack, visiting); + if (*minworld < min_valid2) + *minworld = min_valid2; + if (*minworld > max_valid2) + max_valid2 = 0; if (*maxworld > max_valid2) { cause = callee; *maxworld = max_valid2; @@ -947,12 +961,17 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, size_ // cycle as also having a failed edge. while (stack->len >= depth) { jl_code_instance_t *child = (jl_code_instance_t*)arraylist_pop(stack); - if (*maxworld != jl_atomic_load_relaxed(&child->max_world)) - jl_atomic_store_relaxed(&child->max_world, *maxworld); + if (jl_atomic_load_relaxed(&jl_n_threads) == 1) { + assert(jl_atomic_load_relaxed(&child->max_world) == WORLD_AGE_REVALIDATION_SENTINEL); + assert(*minworld <= jl_atomic_load_relaxed(&child->min_world)); + } + if (*maxworld != 0) + jl_atomic_store_relaxed(&child->min_world, *minworld); + jl_atomic_store_relaxed(&child->max_world, *maxworld); void **bp = ptrhash_bp(visiting, codeinst); assert(*bp == (char*)HT_NOTFOUND + stack->len + 1); *bp = HT_NOTFOUND; - if (_jl_debug_method_invalidation && *maxworld != ~(size_t)0) { + if (_jl_debug_method_invalidation && *maxworld < current_world) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)child); loctag = jl_cstr_to_string("verify_methods"); JL_GC_PUSH1(&loctag); @@ -966,26 +985,30 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, size_ return 0; } -static size_t jl_verify_method_graph(jl_code_instance_t *codeinst, size_t minworld, arraylist_t *stack, htable_t *visiting) +static void jl_verify_method_graph(jl_code_instance_t *codeinst, arraylist_t *stack, htable_t *visiting) { + size_t minworld; + size_t maxworld; assert(stack->len == 0); for (size_t i = 0, hsz = visiting->size; i < hsz; i++) assert(visiting->table[i] == HT_NOTFOUND); - size_t maxworld = ~(size_t)0; - int child_cycle = jl_verify_method(codeinst, minworld, &maxworld, stack, visiting); + int child_cycle = jl_verify_method(codeinst, &minworld, &maxworld, stack, visiting); assert(child_cycle == 0); (void)child_cycle; assert(stack->len == 0); for (size_t i = 0, hsz = visiting->size / 2; i < hsz; i++) { assert(visiting->table[2 * i + 1] == HT_NOTFOUND); visiting->table[2 * i] = HT_NOTFOUND; } - return maxworld; + if (jl_atomic_load_relaxed(&jl_n_threads) == 1) { // a different thread might simultaneously come to a different, but equally valid, alternative result + assert(maxworld == 0 || jl_atomic_load_relaxed(&codeinst->min_world) == minworld); + assert(jl_atomic_load_relaxed(&codeinst->max_world) == maxworld); + } } // Restore backedges to external targets // `edges` = [caller1, ...], the list of worklist-owned code instances internally // `ext_ci_list` = [caller1, ...], the list of worklist-owned code instances externally -static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list, size_t minworld) +static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list) { // determine which CodeInstance objects are still valid in our image // to enable any applicable new codes @@ -1001,53 +1024,51 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list, size jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_array_ptr_ref(edges, i); jl_svec_t *callees = jl_atomic_load_relaxed(&codeinst->edges); jl_method_instance_t *caller = codeinst->def; - if (jl_atomic_load_relaxed(&codeinst->min_world) != minworld) { - if (external && jl_atomic_load_relaxed(&codeinst->max_world) != WORLD_AGE_REVALIDATION_SENTINEL) { - assert(jl_atomic_load_relaxed(&codeinst->min_world) == 1); - assert(jl_atomic_load_relaxed(&codeinst->max_world) == ~(size_t)0); - } - else { - continue; - } - } - size_t maxvalid = jl_verify_method_graph(codeinst, minworld, &stack, &visiting); - assert(jl_atomic_load_relaxed(&codeinst->max_world) == maxvalid); - if (maxvalid == ~(size_t)0) { - // if this callee is still valid, add all the backedges - for (size_t j = 0; j < jl_svec_len(callees); ) { - jl_value_t *edge = jl_svecref(callees, j); - if (jl_is_long(edge)) { - j += 2; // skip over signature and count but not methods - continue; - } - else if (jl_is_method(edge)) { - j += 1; - continue; - } - if (jl_is_code_instance(edge)) - edge = (jl_value_t*)((jl_code_instance_t*)edge)->def; - if (jl_is_method_instance(edge)) { - jl_method_instance_add_backedge((jl_method_instance_t*)edge, NULL, codeinst); - j += 1; - } - else if (jl_is_mtable(edge)) { - jl_methtable_t *mt = (jl_methtable_t*)edge; - jl_value_t *sig = jl_svecref(callees, j + 1); - jl_method_table_add_backedge(mt, sig, codeinst); - j += 2; - } - else { - jl_value_t *callee = jl_svecref(callees, j + 1); - if (jl_is_code_instance(callee)) - callee = (jl_value_t*)((jl_code_instance_t*)callee)->def; - else if (jl_is_method(callee)) { - j += 2; + jl_verify_method_graph(codeinst, &stack, &visiting); + size_t minvalid = jl_atomic_load_relaxed(&codeinst->min_world); + size_t maxvalid = jl_atomic_load_relaxed(&codeinst->max_world); + if (maxvalid >= minvalid) { + if (jl_atomic_load_relaxed(&jl_world_counter) == maxvalid) { + // if this callee is still valid, add all the backedges + for (size_t j = 0; j < jl_svec_len(callees); ) { + jl_value_t *edge = jl_svecref(callees, j); + if (jl_is_long(edge)) { + j += 2; // skip over signature and count but not methods continue; } - jl_method_instance_add_backedge((jl_method_instance_t*)callee, edge, codeinst); - j += 2; + else if (jl_is_method(edge)) { + j += 1; + continue; + } + if (jl_is_code_instance(edge)) + edge = (jl_value_t*)((jl_code_instance_t*)edge)->def; + if (jl_is_method_instance(edge)) { + jl_method_instance_add_backedge((jl_method_instance_t*)edge, NULL, codeinst); + j += 1; + } + else if (jl_is_mtable(edge)) { + jl_methtable_t *mt = (jl_methtable_t*)edge; + jl_value_t *sig = jl_svecref(callees, j + 1); + jl_method_table_add_backedge(mt, sig, codeinst); + j += 2; + } + else { + jl_value_t *callee = jl_svecref(callees, j + 1); + if (jl_is_code_instance(callee)) + callee = (jl_value_t*)((jl_code_instance_t*)callee)->def; + else if (jl_is_method(callee)) { + j += 2; + continue; + } + jl_method_instance_add_backedge((jl_method_instance_t*)callee, edge, codeinst); + j += 2; + } } } + if (jl_atomic_load_relaxed(&jl_world_counter) == maxvalid) { + maxvalid = ~(size_t)0; + jl_atomic_store_relaxed(&codeinst->max_world, maxvalid); + } if (external) { jl_value_t *owner = codeinst->owner; JL_GC_PROMISE_ROOTED(owner); @@ -1055,7 +1076,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list, size // See #53586, #53109 assert(jl_atomic_load_relaxed(&codeinst->inferred)); - if (jl_rettype_inferred(owner, caller, minworld, maxvalid) != jl_nothing) { + if (jl_rettype_inferred(owner, caller, minvalid, maxvalid) != jl_nothing) { // We already got a code instance for this world age range from somewhere else - we don't need // this one. }