Skip to content

Commit

Permalink
8344035: Replace predicate walking code in Loop Unswitching with a pr…
Browse files Browse the repository at this point in the history 
…edicate visitor
  • Loading branch information
@chhagedorn
chhagedorn committed Nov 12, 2024
1 parent 537251f commit 728d918
Show file tree
Hide file tree
Showing 5 changed files with 139 additions and 165 deletions.
135 changes: 0 additions & 135 deletions src/hotspot/share/opto/loopPredicate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,63 +272,6 @@ void PhaseIdealLoop::fix_cloned_data_node_controls(const ProjNode* old_uncommon_
orig_to_clone.iterate_all(orig_clone_action);
}

IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj,
Node* new_entry, Deoptimization::DeoptReason reason,
const bool slow_loop) {

IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate,
slow_loop);
assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection");
ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate();
return new_predicate_proj;
}

// Clones Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its control inputs.
// It also rewires the control edges of data nodes with dependencies in the loop from the old predicates to the new
// cloned predicates.
void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
ParsePredicateSuccessProj* old_parse_predicate_proj,
ParsePredicateNode* true_path_loop_parse_predicate,
ParsePredicateNode* false_path_loop_parse_predicate) {
// Push the original Template Assertion Predicates on a list to later process them in reverse order to keep the
// original predicate order.
Unique_Node_List list;
get_template_assertion_predicates(old_parse_predicate_proj, list);

Node_List to_process;
for (int i = list.size() - 1; i >= 0; i--) {
IfTrueNode* template_assertion_predicate_success_proj = list.at(i)->as_IfTrue();
assert(template_assertion_predicate_success_proj->in(0)->is_If(), "must be If node");

IfTrueNode* true_path_loop_proj =
clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
true_path_loop_parse_predicate);
IfTrueNode* false_path_loop_proj =
clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
false_path_loop_parse_predicate);

// Update control dependent data nodes.
for (DUIterator j = template_assertion_predicate_success_proj->outs();
template_assertion_predicate_success_proj->has_out(j);
j++) {
Node* fast_node = template_assertion_predicate_success_proj->out(j);
if (loop->is_member(get_loop(ctrl_or_self(fast_node)))) {
assert(fast_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
Node* slow_node = old_new[fast_node->_idx];
assert(slow_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
_igvn.replace_input_of(fast_node, 0, true_path_loop_proj);
to_process.push(slow_node);
--j;
}
}
// Have to delay updates to the slow loop so uses of predicate are not modified while we iterate on them.
while (to_process.size() > 0) {
Node* slow_node = to_process.pop();
_igvn.replace_input_of(slow_node, 0, false_path_loop_proj);
}
}
}

// Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead
// of the projections.
Expand All @@ -340,84 +283,6 @@ void PhaseIdealLoop::get_template_assertion_predicates(ParsePredicateSuccessProj
predicate_iterator.for_each(template_assertion_predicate_collector);
}

// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
// predicate again).
IfTrueNode*
PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
ParsePredicateNode* unswitched_loop_parse_predicate) {
TemplateAssertionPredicate template_assertion_predicate(template_assertion_predicate_success_proj);
IfTrueNode* template_success_proj = template_assertion_predicate.clone(unswitched_loop_parse_predicate->in(0), this);
assert(assertion_predicate_has_loop_opaque_node(template_success_proj->in(0)->as_If()),
"must find Assertion Predicate for fast loop");
_igvn.replace_input_of(unswitched_loop_parse_predicate, 0, template_success_proj);
set_idom(unswitched_loop_parse_predicate, template_success_proj, dom_depth(template_success_proj));
return template_success_proj;
}

// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the
// unswitch If.
void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry) {
LoopNode* head = loop->_head->as_Loop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);

const Predicates predicates(entry);
clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(),
Deoptimization::Reason_predicate, true_path_loop_entry, false_path_loop_entry);
clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(),
Deoptimization::Reason_profile_predicate, true_path_loop_entry, false_path_loop_entry);

const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) {
// Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate
// is only created when converting a LoopNode to a CountedLoopNode).
clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check,
true_path_loop_entry, false_path_loop_entry);
}
}

// Clone the Parse Predicate and Template Assertion Predicates of a Loop Predication related Predicate Block.
void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason,
IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry) {
if (predicate_block->has_parse_predicate()) {
// We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will
// be changed with the complete fix for Assertion Predicates.
clone_parse_predicate_to_unswitched_loops(predicate_block, reason, true_path_loop_entry, false_path_loop_entry);
assert(true_path_loop_entry->in(0)->is_ParsePredicate() && false_path_loop_entry->in(0)->is_ParsePredicate(),
"must be success projections of the cloned Parse Predicates");
clone_assertion_predicates_to_unswitched_loop(loop, old_new, predicate_block->parse_predicate_success_proj(),
true_path_loop_entry->in(0)->as_ParsePredicate(),
false_path_loop_entry->in(0)->as_ParsePredicate());
}
}

void PhaseIdealLoop::clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) {
assert(predicate_block->has_parse_predicate(), "must have parse predicate");
ParsePredicateSuccessProj* parse_predicate_proj = predicate_block->parse_predicate_success_proj();
iffast_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, iffast_pred, reason, false);
check_cloned_parse_predicate_for_unswitching(iffast_pred, true);

ifslow_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, ifslow_pred, reason, true);
check_cloned_parse_predicate_for_unswitching(ifslow_pred, false);
}

#ifndef PRODUCT
void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) {
assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate");
if (TraceLoopPredicate) {
tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow");
new_entry->in(0)->dump();
}
}
#endif

//------------------------------Invariance-----------------------------------
// Helper class for loop_predication_impl to compute invariance on the fly and
// clone invariants.
Expand Down
25 changes: 16 additions & 9 deletions src/hotspot/share/opto/loopUnswitch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,12 +237,14 @@ class OriginalLoop : public StackObj {
NONCOPYABLE(OriginalLoop);

private:
void fix_loop_entries(IfProjNode* true_path_loop_entry, IfProjNode* false_path_loop_entry) {
_phase->replace_loop_entry(_loop_head, true_path_loop_entry);
void fix_loop_entries(const CloneUnswitchedLoopPredicatesVisitor& clone_unswitched_loop_predicates_visitor) {
_phase->replace_loop_entry(_loop_head, clone_unswitched_loop_predicates_visitor.new_true_path_loop_entry());
LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop();
_phase->replace_loop_entry(false_path_loop_strip_mined_head, false_path_loop_entry);
_phase->replace_loop_entry(false_path_loop_strip_mined_head,
clone_unswitched_loop_predicates_visitor.new_false_path_loop_entry());
}


Node* old_to_new(const Node* old) const {
return _old_new[old->_idx];
}
Expand Down Expand Up @@ -280,24 +282,29 @@ class OriginalLoop : public StackObj {
// Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop.
// Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If.
void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) {
const uint first_slow_loop_node_index = _phase->C->unique();
_phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector());

// At this point, the selector If projections are the corresponding loop entries.
// clone_parse_and_assertion_predicates_to_unswitched_loop() could clone additional predicates after the selector
// If projections. The loop entries are updated accordingly.
IfProjNode* true_path_loop_entry = unswitched_loop_selector.true_path_loop_proj();
IfProjNode* false_path_loop_entry = unswitched_loop_selector.false_path_loop_proj();
_phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, _old_new,
true_path_loop_entry, false_path_loop_entry);

fix_loop_entries(true_path_loop_entry, false_path_loop_entry);
CloneUnswitchedLoopPredicatesVisitor
clone_unswitched_loop_predicates_visitor(unswitched_loop_selector.true_path_loop_proj(),
unswitched_loop_selector.false_path_loop_proj(),
first_slow_loop_node_index, _old_new, _phase);
move_predicates(clone_unswitched_loop_predicates_visitor);

DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);)

_phase->recompute_dom_depth();
remove_unswitch_candidate_from_loops(unswitched_loop_selector);
}

void move_predicates(CloneUnswitchedLoopPredicatesVisitor& clone_predicates_for_unswitched_loops) const {
PredicateIterator predicate_iterator(_loop_head->in(LoopNode::EntryControl));
predicate_iterator.for_each(clone_predicates_for_unswitched_loops);
}
};

// See comments below file header for more information about Loop Unswitching.
Expand Down
21 changes: 0 additions & 21 deletions src/hotspot/share/opto/loopnode.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1656,28 +1656,7 @@ class PhaseIdealLoop : public PhaseTransform {
_nodes_required = UINT_MAX;
}

public:
// Clone Parse Predicates to slow and fast loop when unswitching a loop
void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry);
private:
void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason,
IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry);
void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, bool slow_loop);
void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
ParsePredicateSuccessProj* old_parse_predicate_proj,
ParsePredicateNode* true_path_loop_parse_predicate,
ParsePredicateNode* false_path_loop_parse_predicate);
IfTrueNode* clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
ParsePredicateNode* unswitched_loop_parse_predicate);
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;

bool _created_loop_node;
DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);)
Expand Down
Loading

0 comments on commit 728d918

Please sign in to comment.