From 550933659a8021131d9d1424fc6ff77b51745cbe Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Tue, 12 Nov 2024 15:57:47 +0100 Subject: [PATCH] 8344035: Replace predicate walking code in Loop Unswitching with a predicate visitor --- src/hotspot/share/opto/loopPredicate.cpp | 145 +---------------------- src/hotspot/share/opto/loopUnswitch.cpp | 67 ++++++----- src/hotspot/share/opto/loopnode.cpp | 4 +- src/hotspot/share/opto/loopnode.hpp | 24 +--- src/hotspot/share/opto/predicates.cpp | 101 +++++++++++++++- src/hotspot/share/opto/predicates.hpp | 80 +++++++++++-- 6 files changed, 214 insertions(+), 207 deletions(-) diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 9340f7e097846..8246d07798eda 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -272,150 +272,15 @@ void PhaseIdealLoop::fix_cloned_data_node_controls(const ProjNode* old_uncommon_ orig_to_clone.iterate_all(orig_clone_action); } -IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, - Node* new_entry, Deoptimization::DeoptReason reason, - const bool slow_loop) { - - IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate, - slow_loop); - assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection"); - ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate(); - return new_predicate_proj; -} - -// Clones Template Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its -// control inputs. It also rewires the control edges of data nodes with dependencies in the loop from the old predicates -// to the new cloned predicates. -void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - ParsePredicateSuccessProj* old_parse_predicate_proj, - ParsePredicateNode* true_path_loop_parse_predicate, - ParsePredicateNode* false_path_loop_parse_predicate) { - // Push the original Template Assertion Predicates on a list to later process them in reverse order to keep the - // original predicate order. - Unique_Node_List list; - get_template_assertion_predicates(old_parse_predicate_proj, list); - - Node_List to_process; - for (int i = list.size() - 1; i >= 0; i--) { - IfTrueNode* template_assertion_predicate_success_proj = list.at(i)->as_IfTrue(); - assert(template_assertion_predicate_success_proj->in(0)->is_If(), "must be If node"); - - IfTrueNode* true_path_loop_proj = - clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj, - true_path_loop_parse_predicate); - IfTrueNode* false_path_loop_proj = - clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj, - false_path_loop_parse_predicate); - - // Update control dependent data nodes. - for (DUIterator j = template_assertion_predicate_success_proj->outs(); - template_assertion_predicate_success_proj->has_out(j); - j++) { - Node* true_path_loop_node = template_assertion_predicate_success_proj->out(j); - if (loop->is_member(get_loop(ctrl_or_self(true_path_loop_node)))) { - assert(true_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge"); - Node* false_path_loop_node = old_new[true_path_loop_node->_idx]; - assert(false_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge"); - _igvn.replace_input_of(true_path_loop_node, 0, true_path_loop_proj); - to_process.push(false_path_loop_node); - --j; - } - } - // Have to delay updates to the false path loop so uses of predicate are not modified while we iterate on them. - while (to_process.size() > 0) { - Node* slow_node = to_process.pop(); - _igvn.replace_input_of(slow_node, 0, false_path_loop_proj); - } - } -} - -// Put all Template Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque' -// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead -// of the projections. -void PhaseIdealLoop::get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, - const bool get_opaque) { +// Put all OpaqueTemplateAssertionPredicate nodes on a list, starting at 'predicate' and going up in the tree. +void PhaseIdealLoop::get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj, + Unique_Node_List& list) { Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason(); PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason); - TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque); - predicate_iterator.for_each(template_assertion_predicate_collector); -} - -// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon -// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned -// predicate again). -IfTrueNode* -PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj, - ParsePredicateNode* unswitched_loop_parse_predicate) { - TemplateAssertionPredicate template_assertion_predicate(template_assertion_predicate_success_proj); - IfTrueNode* template_success_proj = template_assertion_predicate.clone(unswitched_loop_parse_predicate->in(0), this); - _igvn.replace_input_of(unswitched_loop_parse_predicate, 0, template_success_proj); - set_idom(unswitched_loop_parse_predicate, template_success_proj, dom_depth(template_success_proj)); - return template_success_proj; -} - -// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the -// unswitch If. -void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, - IfProjNode*& true_path_loop_entry, - IfProjNode*& false_path_loop_entry) { - LoopNode* head = loop->_head->as_Loop(); - Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - - const Predicates predicates(entry); - clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(), - Deoptimization::Reason_predicate, true_path_loop_entry, false_path_loop_entry); - clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(), - Deoptimization::Reason_profile_predicate, true_path_loop_entry, false_path_loop_entry); - - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) { - // Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate - // is only created when converting a LoopNode to a CountedLoopNode). - clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check, - true_path_loop_entry, false_path_loop_entry); - } -} - -// Clone the Parse Predicate and Template Assertion Predicates of a Loop Predication related Predicate Block. -void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, - IfProjNode*& true_path_loop_entry, - IfProjNode*& false_path_loop_entry) { - if (predicate_block->has_parse_predicate()) { - // We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will - // be changed with the complete fix for Assertion Predicates. - clone_parse_predicate_to_unswitched_loops(predicate_block, reason, true_path_loop_entry, false_path_loop_entry); - assert(true_path_loop_entry->in(0)->is_ParsePredicate() && false_path_loop_entry->in(0)->is_ParsePredicate(), - "must be success projections of the cloned Parse Predicates"); - clone_assertion_predicates_to_unswitched_loop(loop, old_new, predicate_block->parse_predicate_success_proj(), - true_path_loop_entry->in(0)->as_ParsePredicate(), - false_path_loop_entry->in(0)->as_ParsePredicate()); - } + OpaqueTemplateAssertionPredicateCollector opaque_template_assertion_predicate_collector(list); + predicate_iterator.for_each(opaque_template_assertion_predicate_collector); } -void PhaseIdealLoop::clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, - IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) { - assert(predicate_block->has_parse_predicate(), "must have parse predicate"); - ParsePredicateSuccessProj* parse_predicate_proj = predicate_block->parse_predicate_success_proj(); - iffast_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, iffast_pred, reason, false); - check_cloned_parse_predicate_for_unswitching(iffast_pred, true); - - ifslow_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, ifslow_pred, reason, true); - check_cloned_parse_predicate_for_unswitching(ifslow_pred, false); -} - -#ifndef PRODUCT -void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) { - assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate"); - if (TraceLoopPredicate) { - tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow"); - new_entry->in(0)->dump(); - } -} -#endif - //------------------------------Invariance----------------------------------- // Helper class for loop_predication_impl to compute invariance on the fly and // clone invariants. diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp index 05623aefaa94a..e93f1d3b8065d 100644 --- a/src/hotspot/share/opto/loopUnswitch.cpp +++ b/src/hotspot/share/opto/loopUnswitch.cpp @@ -236,18 +236,47 @@ class OriginalLoop : public StackObj { _phase(loop->_phase) {} NONCOPYABLE(OriginalLoop); + // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop. + // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If. + void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) { + const uint first_false_path_loop_node_index = _phase->C->unique(); + clone_loop(unswitched_loop_selector); + + move_parse_and_template_assertion_predicates_to_unswitched_loops(first_false_path_loop_node_index); + DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);) + + _phase->recompute_dom_depth(); + remove_unswitch_candidate_from_loops(unswitched_loop_selector); + } + private: - void fix_loop_entries(IfProjNode* true_path_loop_entry, IfProjNode* false_path_loop_entry) { - _phase->replace_loop_entry(_loop_head, true_path_loop_entry); + void clone_loop(const UnswitchedLoopSelector& unswitched_loop_selector) { + _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head), + PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector()); + fix_loop_entries(unswitched_loop_selector); + } + + void fix_loop_entries(const UnswitchedLoopSelector& unswitched_loop_selector) { + _phase->replace_loop_entry(_loop_head, unswitched_loop_selector.true_path_loop_proj()); LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop(); - _phase->replace_loop_entry(false_path_loop_strip_mined_head, false_path_loop_entry); + _phase->replace_loop_entry(false_path_loop_strip_mined_head, + unswitched_loop_selector.false_path_loop_proj()); } - Node* old_to_new(const Node* old) const { - return _old_new[old->_idx]; + // Moves the Parse And Template Assertion Predicates to the true and false path loop. They are inserted between the + // loop heads and the loop selector If projections. The old Parse and Template Assertion Predicates before + // the unswitched loop selector are killed. + void move_parse_and_template_assertion_predicates_to_unswitched_loops(const uint first_false_path_loop_node_index) const { + const NodeInOriginalLoopBody node_in_true_path_loop_body(first_false_path_loop_node_index, _old_new); + const NodeInClonedLoopBody node_in_false_path_loop_body(first_false_path_loop_node_index); + CloneUnswitchedLoopPredicatesVisitor + clone_unswitched_loop_predicates_visitor(_loop_head, old_to_new(_loop_head)->as_Loop(), node_in_true_path_loop_body, + node_in_false_path_loop_body, _phase); + PredicateIterator predicate_iterator(_loop_head->in(LoopNode::EntryControl)); + predicate_iterator.for_each(clone_unswitched_loop_predicates_visitor); } -#ifdef ASSERT + #ifdef ASSERT void verify_unswitched_loop_versions(LoopNode* true_path_loop_head, const UnswitchedLoopSelector& unswitched_loop_selector) const { verify_unswitched_loop_version(true_path_loop_head, unswitched_loop_selector.true_path_loop_proj()); @@ -263,6 +292,10 @@ class OriginalLoop : public StackObj { } #endif // ASSERT + Node* old_to_new(const Node* old) const { + return _old_new[old->_idx]; + } + // Remove the unswitch candidate If nodes in both unswitched loop versions which are now dominated by the loop selector // If node. Keep the true-path-path in the true-path-loop and the false-path-path in the false-path-loop by setting // the bool input accordingly. The unswitch candidate If nodes are folded in the next IGVN round. @@ -276,28 +309,6 @@ class OriginalLoop : public StackObj { _phase->dominated_by(unswitched_loop_selector.false_path_loop_proj(), unswitching_candidate_clone); } - public: - // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop. - // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If. - void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) { - _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head), - PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector()); - - // At this point, the selector If projections are the corresponding loop entries. - // clone_parse_and_assertion_predicates_to_unswitched_loop() could clone additional predicates after the selector - // If projections. The loop entries are updated accordingly. - IfProjNode* true_path_loop_entry = unswitched_loop_selector.true_path_loop_proj(); - IfProjNode* false_path_loop_entry = unswitched_loop_selector.false_path_loop_proj(); - _phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, _old_new, - true_path_loop_entry, false_path_loop_entry); - - fix_loop_entries(true_path_loop_entry, false_path_loop_entry); - - DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);) - - _phase->recompute_dom_depth(); - remove_unswitch_candidate_from_loops(unswitched_loop_selector); - } }; // See comments below file header for more information about Loop Unswitching. diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index e52f31b1e118d..09e4e747721a3 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4470,7 +4470,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); if (profiled_loop_predicate_block->has_parse_predicate()) { ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); - get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true); + get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates); } } @@ -4478,7 +4478,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); if (loop_predicate_block->has_parse_predicate()) { ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); - get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true); + get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates); } } } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 326a7ba8c8a69..5e036d3d46a7c 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -942,7 +942,8 @@ class PhaseIdealLoop : public PhaseTransform { static void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop); #endif private: - static void get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false); + static void get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj, + Unique_Node_List& list); void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head); void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, CountedLoopNode* remaining_loop_head, @@ -1657,28 +1658,7 @@ class PhaseIdealLoop : public PhaseTransform { _nodes_required = UINT_MAX; } - public: - // Clone Parse Predicates to slow and fast loop when unswitching a loop - void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, - IfProjNode*& true_path_loop_entry, - IfProjNode*& false_path_loop_entry); private: - void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - const PredicateBlock* predicate_block, - Deoptimization::DeoptReason reason, - IfProjNode*& true_path_loop_entry, - IfProjNode*& false_path_loop_entry); - void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason, - IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred); - IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, - Deoptimization::DeoptReason reason, bool slow_loop); - void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, - ParsePredicateSuccessProj* old_parse_predicate_proj, - ParsePredicateNode* true_path_loop_parse_predicate, - ParsePredicateNode* false_path_loop_parse_predicate); - IfTrueNode* clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj, - ParsePredicateNode* unswitched_loop_parse_predicate); - static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN; bool _created_loop_node; DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);) diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index c944506c547ee..5be0db2699c39 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -82,6 +82,25 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p return nullptr; } +ParsePredicateNode* ParsePredicate::clone_to_unswitched_loop(Node* new_control, const bool is_false_path_loop, + PhaseIdealLoop* phase) const { + ParsePredicateSuccessProj* success_proj = phase->create_new_if_for_predicate(_success_proj, new_control, + _parse_predicate_node->deopt_reason(), + Op_ParsePredicate, is_false_path_loop); + NOT_PRODUCT(trace_cloned_parse_predicate(is_false_path_loop, success_proj)); + return success_proj->in(0)->as_ParsePredicate(); +} + +#ifndef PRODUCT +void ParsePredicate::trace_cloned_parse_predicate(const bool is_false_path_loop, + const ParsePredicateSuccessProj* success_proj) { + if (TraceLoopPredicate) { + tty->print("Parse Predicate cloned to %s path loop: ", is_false_path_loop ? "false" : "true"); + success_proj->in(0)->dump(); + } +} +#endif // NOT PRODUCT + Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) { CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); if (uct_call == nullptr) { @@ -196,6 +215,13 @@ IfTrueNode* TemplateAssertionPredicate::initialize(PhaseIdealLoop* phase, Node* return success_proj; } +// Kills the Template Assertion Predicate by setting the condition to true. Will be folded away in the next IGVN round. +void TemplateAssertionPredicate::kill(PhaseIdealLoop* phase) const { + ConINode* true_con = phase->igvn().intcon(1); + phase->set_ctrl(true_con, phase->C->root()); + phase->igvn().replace_input_of(_if_node, 1, true_con); +} + #ifdef ASSERT // Class to verify Initialized and Template Assertion Predicates by trying to find OpaqueLoop*Nodes. class OpaqueLoopNodesVerifier : public BFSActions { @@ -879,6 +905,41 @@ void Predicates::dump_for_loop(LoopNode* loop_node) { } #endif // NOT PRODUCT +TargetLoopPredicateChain::TargetLoopPredicateChain(LoopNode* loop_head, const NodeInLoopBody& node_in_loop_body, + PhaseIdealLoop* phase) + : _loop_selector_proj(loop_head->in(LoopNode::EntryControl)), + _node_in_loop_body(node_in_loop_body), + _phase(phase) {} + +// Clones the provided Template Assertion Predicate to the head of the current predicate chain at the target loop. +// +// This is done in two steps: +// +// old source loop entry old target loop entry old target loop entry +// | | | | +// Template Assertion 1. clone | Cloned Template 2. rewire Cloned Template +// Predicate 1 ===> | Assertion Predicate ===> Assertion Predicate +// | | | +// source loop head target loop head target loop head +void TargetLoopPredicateChain::clone_template_assertion_predicate_to_chain( + const TemplateAssertionPredicate& template_assertion_predicate) { + Node* current_predicate_chain_head = predicate_chain_head(); + IfTrueNode* cloned_template_success_proj = template_assertion_predicate.clone(_loop_selector_proj, _phase); + template_assertion_predicate.rewire_loop_data_dependencies(cloned_template_success_proj, _node_in_loop_body, + _phase); + rewire_to_target_chain_head(cloned_template_success_proj->in(0)->as_If(), current_predicate_chain_head); +} + +// Rewires the newly cloned Template Assertion Predicates to the head of the current predicate chain at the target loop. +void TargetLoopPredicateChain::rewire_to_target_chain_head(IfNode* cloned_template_assertion_predicate, + Node* current_predicate_chain_head) const { + if (current_predicate_chain_head->is_Loop()) { + _phase->replace_loop_entry(current_predicate_chain_head->as_Loop(), cloned_template_assertion_predicate); + } else { + _phase->replace_control(current_predicate_chain_head, cloned_template_assertion_predicate); + } +} + CreateAssertionPredicatesVisitor::CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head, PhaseIdealLoop* phase, const NodeInLoopBody& node_in_loop_body, @@ -948,8 +1009,8 @@ IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_in // x // | old target // Template Assertion loop entry -// Predicate 1 old target clone | \ -// | loop entry TAP 2 | cloned Template Assertion +// Predicate 1 old target clone | \ +// | loop entry TAP 2 | cloned Template Assertion // Template Assertion | ======> | Predicate 2 // Predicate 2 target loop | // | target loop #_current_predicate_chain_head @@ -978,6 +1039,42 @@ void CreateAssertionPredicatesVisitor::rewire_to_old_predicate_chain_head( } } +CloneUnswitchedLoopPredicatesVisitor::CloneUnswitchedLoopPredicatesVisitor( + LoopNode* true_path_loop_head, LoopNode* false_path_loop_head, + const NodeInOriginalLoopBody& node_in_true_path_loop_body, const NodeInClonedLoopBody& node_in_false_path_loop_body, + PhaseIdealLoop* phase) + : _true_path_loop_predicate_chain(true_path_loop_head, node_in_true_path_loop_body, phase), + _false_path_loop_predicate_chain(false_path_loop_head, node_in_false_path_loop_body, phase), + _phase(phase), + _has_hoisted_check_parse_predicates(false) {} + +// Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found. +// The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block. +void CloneUnswitchedLoopPredicatesVisitor::visit(const ParsePredicate& parse_predicate) { + Deoptimization::DeoptReason deopt_reason = parse_predicate.head()->deopt_reason(); + if (deopt_reason == Deoptimization::Reason_predicate || + deopt_reason == Deoptimization::Reason_profile_predicate) { + _has_hoisted_check_parse_predicates = true; + } + + _true_path_loop_predicate_chain.clone_parse_predicate_to_chain(parse_predicate, true); + _false_path_loop_predicate_chain.clone_parse_predicate_to_chain(parse_predicate, false); + parse_predicate.kill(_phase->igvn()); +} + +// Clone the Template Assertion Predicate, which is currently found before the newly added unswitched loop selector, +// to the true path and false path loop. +void CloneUnswitchedLoopPredicatesVisitor::visit(const TemplateAssertionPredicate& template_assertion_predicate) { + if (!_has_hoisted_check_parse_predicates) { + // Only process if we are in the correct Predicate Block. + return; + } + + _true_path_loop_predicate_chain.clone_template_assertion_predicate_to_chain(template_assertion_predicate); + _false_path_loop_predicate_chain.clone_template_assertion_predicate_to_chain(template_assertion_predicate); + template_assertion_predicate.kill(_phase); +} + // Clone the Template Assertion Predicate and set a new input for the OpaqueLoopStrideNode. void UpdateStrideForAssertionPredicates::visit(const TemplateAssertionPredicate& template_assertion_predicate) { if (!template_assertion_predicate.is_last_value()) { diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index 4cd909376aa0d..e5b939d0b2f1c 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -232,7 +232,7 @@ class Predicate : public StackObj { // Generic predicate visitor that does nothing. Subclass this visitor to add customized actions for each predicate. // The visit methods of this visitor are called from the predicate iterator classes which walk the predicate chain. // Use the UnifiedPredicateVisitor if the type of the predicate does not matter. -class PredicateVisitor : StackObj { +class PredicateVisitor : public StackObj { public: virtual void visit(const ParsePredicate& parse_predicate) {} virtual void visit(const RuntimePredicate& runtime_predicate) {} @@ -297,6 +297,8 @@ class ParsePredicate : public Predicate { } static ParsePredicateNode* init_parse_predicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason); + NOT_PRODUCT(static void trace_cloned_parse_predicate(bool is_false_path_loop, + const ParsePredicateSuccessProj* success_proj);) public: ParsePredicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason) @@ -325,6 +327,14 @@ class ParsePredicate : public Predicate { assert(is_valid(), "must be valid"); return _success_proj; } + + ParsePredicateNode* clone_to_unswitched_loop(Node* new_control, bool is_false_path_loop, PhaseIdealLoop* phase) const; + + // Kills this Parse Predicate by marking it useless. Will be folded away in the next IGVN round. + void kill(PhaseIterGVN& igvn) const { + _parse_predicate_node->mark_useless(); + igvn._worklist.push(_parse_predicate_node); + } }; // Class to represent a Runtime Predicate which always has an associated UCT on the failing path. @@ -399,6 +409,7 @@ class TemplateAssertionPredicate : public Predicate { IfTrueNode* initialize(PhaseIdealLoop* phase, Node* new_control) const; void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& data_in_loop_body, PhaseIdealLoop* phase) const; + void kill(PhaseIdealLoop* phase) const; static bool is_predicate(Node* node); #ifdef ASSERT @@ -978,6 +989,31 @@ class NodeInClonedLoopBody : public NodeInLoopBody { } }; +// This class can be used to create predicates at a target loop where a new chain of predicates need to be established. +class TargetLoopPredicateChain : public StackObj { + Node* const _loop_selector_proj; + const NodeInLoopBody& _node_in_loop_body; + PhaseIdealLoop* const _phase; + + Node* predicate_chain_head() const { + return _loop_selector_proj->unique_ctrl_out(); + } + + void rewire_to_target_chain_head(IfNode* cloned_template_assertion_predicate, Node* current_predicate_chain_head) const; + + public: + TargetLoopPredicateChain(LoopNode* loop_head, const NodeInLoopBody& node_in_loop_body, PhaseIdealLoop* phase); + NONCOPYABLE(TargetLoopPredicateChain); + + // Clones the provided Parse Predicate to the head of the current predicate chain at the target loop. + // This method is used for Loop Unswitching. + void clone_parse_predicate_to_chain(const ParsePredicate& parse_predicate, bool is_true_path_loop) { + parse_predicate.clone_to_unswitched_loop(_loop_selector_proj, !is_true_path_loop, _phase); + } + + void clone_template_assertion_predicate_to_chain(const TemplateAssertionPredicate& template_assertion_predicate); +}; + // Visitor to create Initialized Assertion Predicates at a target loop from Template Assertion Predicates from a source // loop. This visitor can be used in combination with a PredicateIterator. class CreateAssertionPredicatesVisitor : public PredicateVisitor { @@ -1006,25 +1042,43 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor { void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; }; -// This visitor collects all Template Assertion Predicates If nodes or the corresponding Opaque nodes, depending on the -// provided 'get_opaque' flag, to the provided list. -class TemplateAssertionPredicateCollector : public PredicateVisitor { +// Visitor to clone Parse and Template Assertion Predicates from a loop to its unswitched true and false path loop. +// The cloned predicates are not updated in any way. Thus, an Initialized Assertion Predicate is also not required to +// be created. Note that the data dependencies from the Template Assertion Predicates are also updated to the newly +// cloned Templates, depending on whether they belong to the true or false path loop. +class CloneUnswitchedLoopPredicatesVisitor : public PredicateVisitor { + TargetLoopPredicateChain _true_path_loop_predicate_chain; + TargetLoopPredicateChain _false_path_loop_predicate_chain; + + PhaseIdealLoop* const _phase; + bool _has_hoisted_check_parse_predicates; + + public: + CloneUnswitchedLoopPredicatesVisitor(LoopNode* true_path_loop_head, + LoopNode* false_path_loop_head, + const NodeInOriginalLoopBody& node_in_true_path_loop_body, + const NodeInClonedLoopBody& node_in_false_path_loop_body, + PhaseIdealLoop* phase); + NONCOPYABLE(CloneUnswitchedLoopPredicatesVisitor); + + using PredicateVisitor::visit; + + void visit(const ParsePredicate& parse_predicate) override; + void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; +}; + +// This visitor collects all OpaqueTemplateAssertionNodes of Template Assertion Predicates. This is used for cleaning +// up unused Template Assertion Predicates. +class OpaqueTemplateAssertionPredicateCollector : public PredicateVisitor { Unique_Node_List& _list; - const bool _get_opaque; public: - TemplateAssertionPredicateCollector(Unique_Node_List& list, const bool get_opaque) - : _list(list), - _get_opaque(get_opaque) {} + explicit OpaqueTemplateAssertionPredicateCollector(Unique_Node_List& list) : _list(list) {} using PredicateVisitor::visit; void visit(const TemplateAssertionPredicate& template_assertion_predicate) override { - if (_get_opaque) { - _list.push(template_assertion_predicate.opaque_node()); - } else { - _list.push(template_assertion_predicate.tail()); - } + _list.push(template_assertion_predicate.opaque_node()); } };