diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 8246d07798eda..9340f7e097846 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -272,15 +272,150 @@ void PhaseIdealLoop::fix_cloned_data_node_controls(const ProjNode* old_uncommon_ orig_to_clone.iterate_all(orig_clone_action); } -// Put all OpaqueTemplateAssertionPredicate nodes on a list, starting at 'predicate' and going up in the tree. -void PhaseIdealLoop::get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj, - Unique_Node_List& list) { +IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, + Node* new_entry, Deoptimization::DeoptReason reason, + const bool slow_loop) { + + IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate, + slow_loop); + assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection"); + ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate(); + return new_predicate_proj; +} + +// Clones Template Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its +// control inputs. It also rewires the control edges of data nodes with dependencies in the loop from the old predicates +// to the new cloned predicates. +void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, + ParsePredicateSuccessProj* old_parse_predicate_proj, + ParsePredicateNode* true_path_loop_parse_predicate, + ParsePredicateNode* false_path_loop_parse_predicate) { + // Push the original Template Assertion Predicates on a list to later process them in reverse order to keep the + // original predicate order. + Unique_Node_List list; + get_template_assertion_predicates(old_parse_predicate_proj, list); + + Node_List to_process; + for (int i = list.size() - 1; i >= 0; i--) { + IfTrueNode* template_assertion_predicate_success_proj = list.at(i)->as_IfTrue(); + assert(template_assertion_predicate_success_proj->in(0)->is_If(), "must be If node"); + + IfTrueNode* true_path_loop_proj = + clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj, + true_path_loop_parse_predicate); + IfTrueNode* false_path_loop_proj = + clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj, + false_path_loop_parse_predicate); + + // Update control dependent data nodes. + for (DUIterator j = template_assertion_predicate_success_proj->outs(); + template_assertion_predicate_success_proj->has_out(j); + j++) { + Node* true_path_loop_node = template_assertion_predicate_success_proj->out(j); + if (loop->is_member(get_loop(ctrl_or_self(true_path_loop_node)))) { + assert(true_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge"); + Node* false_path_loop_node = old_new[true_path_loop_node->_idx]; + assert(false_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge"); + _igvn.replace_input_of(true_path_loop_node, 0, true_path_loop_proj); + to_process.push(false_path_loop_node); + --j; + } + } + // Have to delay updates to the false path loop so uses of predicate are not modified while we iterate on them. + while (to_process.size() > 0) { + Node* slow_node = to_process.pop(); + _igvn.replace_input_of(slow_node, 0, false_path_loop_proj); + } + } +} + +// Put all Template Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque' +// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead +// of the projections. +void PhaseIdealLoop::get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, + const bool get_opaque) { Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason(); PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason); - OpaqueTemplateAssertionPredicateCollector opaque_template_assertion_predicate_collector(list); - predicate_iterator.for_each(opaque_template_assertion_predicate_collector); + TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque); + predicate_iterator.for_each(template_assertion_predicate_collector); +} + +// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon +// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned +// predicate again). +IfTrueNode* +PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj, + ParsePredicateNode* unswitched_loop_parse_predicate) { + TemplateAssertionPredicate template_assertion_predicate(template_assertion_predicate_success_proj); + IfTrueNode* template_success_proj = template_assertion_predicate.clone(unswitched_loop_parse_predicate->in(0), this); + _igvn.replace_input_of(unswitched_loop_parse_predicate, 0, template_success_proj); + set_idom(unswitched_loop_parse_predicate, template_success_proj, dom_depth(template_success_proj)); + return template_success_proj; +} + +// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the +// unswitch If. +void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, + IfProjNode*& true_path_loop_entry, + IfProjNode*& false_path_loop_entry) { + LoopNode* head = loop->_head->as_Loop(); + Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); + + const Predicates predicates(entry); + clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(), + Deoptimization::Reason_predicate, true_path_loop_entry, false_path_loop_entry); + clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(), + Deoptimization::Reason_profile_predicate, true_path_loop_entry, false_path_loop_entry); + + const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); + if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) { + // Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate + // is only created when converting a LoopNode to a CountedLoopNode). + clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check, + true_path_loop_entry, false_path_loop_entry); + } +} + +// Clone the Parse Predicate and Template Assertion Predicates of a Loop Predication related Predicate Block. +void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, + const PredicateBlock* predicate_block, + Deoptimization::DeoptReason reason, + IfProjNode*& true_path_loop_entry, + IfProjNode*& false_path_loop_entry) { + if (predicate_block->has_parse_predicate()) { + // We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will + // be changed with the complete fix for Assertion Predicates. + clone_parse_predicate_to_unswitched_loops(predicate_block, reason, true_path_loop_entry, false_path_loop_entry); + assert(true_path_loop_entry->in(0)->is_ParsePredicate() && false_path_loop_entry->in(0)->is_ParsePredicate(), + "must be success projections of the cloned Parse Predicates"); + clone_assertion_predicates_to_unswitched_loop(loop, old_new, predicate_block->parse_predicate_success_proj(), + true_path_loop_entry->in(0)->as_ParsePredicate(), + false_path_loop_entry->in(0)->as_ParsePredicate()); + } } +void PhaseIdealLoop::clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, + Deoptimization::DeoptReason reason, + IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) { + assert(predicate_block->has_parse_predicate(), "must have parse predicate"); + ParsePredicateSuccessProj* parse_predicate_proj = predicate_block->parse_predicate_success_proj(); + iffast_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, iffast_pred, reason, false); + check_cloned_parse_predicate_for_unswitching(iffast_pred, true); + + ifslow_pred = clone_parse_predicate_to_unswitched_loop(parse_predicate_proj, ifslow_pred, reason, true); + check_cloned_parse_predicate_for_unswitching(ifslow_pred, false); +} + +#ifndef PRODUCT +void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) { + assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate"); + if (TraceLoopPredicate) { + tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow"); + new_entry->in(0)->dump(); + } +} +#endif + //------------------------------Invariance----------------------------------- // Helper class for loop_predication_impl to compute invariance on the fly and // clone invariants. diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp index e93f1d3b8065d..05623aefaa94a 100644 --- a/src/hotspot/share/opto/loopUnswitch.cpp +++ b/src/hotspot/share/opto/loopUnswitch.cpp @@ -236,47 +236,18 @@ class OriginalLoop : public StackObj { _phase(loop->_phase) {} NONCOPYABLE(OriginalLoop); - // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop. - // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If. - void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) { - const uint first_false_path_loop_node_index = _phase->C->unique(); - clone_loop(unswitched_loop_selector); - - move_parse_and_template_assertion_predicates_to_unswitched_loops(first_false_path_loop_node_index); - DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);) - - _phase->recompute_dom_depth(); - remove_unswitch_candidate_from_loops(unswitched_loop_selector); - } - private: - void clone_loop(const UnswitchedLoopSelector& unswitched_loop_selector) { - _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head), - PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector()); - fix_loop_entries(unswitched_loop_selector); - } - - void fix_loop_entries(const UnswitchedLoopSelector& unswitched_loop_selector) { - _phase->replace_loop_entry(_loop_head, unswitched_loop_selector.true_path_loop_proj()); + void fix_loop_entries(IfProjNode* true_path_loop_entry, IfProjNode* false_path_loop_entry) { + _phase->replace_loop_entry(_loop_head, true_path_loop_entry); LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop(); - _phase->replace_loop_entry(false_path_loop_strip_mined_head, - unswitched_loop_selector.false_path_loop_proj()); + _phase->replace_loop_entry(false_path_loop_strip_mined_head, false_path_loop_entry); } - // Moves the Parse And Template Assertion Predicates to the true and false path loop. They are inserted between the - // loop heads and the loop selector If projections. The old Parse and Template Assertion Predicates before - // the unswitched loop selector are killed. - void move_parse_and_template_assertion_predicates_to_unswitched_loops(const uint first_false_path_loop_node_index) const { - const NodeInOriginalLoopBody node_in_true_path_loop_body(first_false_path_loop_node_index, _old_new); - const NodeInClonedLoopBody node_in_false_path_loop_body(first_false_path_loop_node_index); - CloneUnswitchedLoopPredicatesVisitor - clone_unswitched_loop_predicates_visitor(_loop_head, old_to_new(_loop_head)->as_Loop(), node_in_true_path_loop_body, - node_in_false_path_loop_body, _phase); - PredicateIterator predicate_iterator(_loop_head->in(LoopNode::EntryControl)); - predicate_iterator.for_each(clone_unswitched_loop_predicates_visitor); + Node* old_to_new(const Node* old) const { + return _old_new[old->_idx]; } - #ifdef ASSERT +#ifdef ASSERT void verify_unswitched_loop_versions(LoopNode* true_path_loop_head, const UnswitchedLoopSelector& unswitched_loop_selector) const { verify_unswitched_loop_version(true_path_loop_head, unswitched_loop_selector.true_path_loop_proj()); @@ -292,10 +263,6 @@ class OriginalLoop : public StackObj { } #endif // ASSERT - Node* old_to_new(const Node* old) const { - return _old_new[old->_idx]; - } - // Remove the unswitch candidate If nodes in both unswitched loop versions which are now dominated by the loop selector // If node. Keep the true-path-path in the true-path-loop and the false-path-path in the false-path-loop by setting // the bool input accordingly. The unswitch candidate If nodes are folded in the next IGVN round. @@ -309,6 +276,28 @@ class OriginalLoop : public StackObj { _phase->dominated_by(unswitched_loop_selector.false_path_loop_proj(), unswitching_candidate_clone); } + public: + // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop. + // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If. + void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) { + _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head), + PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector()); + + // At this point, the selector If projections are the corresponding loop entries. + // clone_parse_and_assertion_predicates_to_unswitched_loop() could clone additional predicates after the selector + // If projections. The loop entries are updated accordingly. + IfProjNode* true_path_loop_entry = unswitched_loop_selector.true_path_loop_proj(); + IfProjNode* false_path_loop_entry = unswitched_loop_selector.false_path_loop_proj(); + _phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, _old_new, + true_path_loop_entry, false_path_loop_entry); + + fix_loop_entries(true_path_loop_entry, false_path_loop_entry); + + DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);) + + _phase->recompute_dom_depth(); + remove_unswitch_candidate_from_loops(unswitched_loop_selector); + } }; // See comments below file header for more information about Loop Unswitching. diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 09e4e747721a3..e52f31b1e118d 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4470,7 +4470,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); if (profiled_loop_predicate_block->has_parse_predicate()) { ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); - get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates); + get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true); } } @@ -4478,7 +4478,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); if (loop_predicate_block->has_parse_predicate()) { ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); - get_opaque_template_assertion_predicate_nodes(parse_predicate_proj, useful_predicates); + get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true); } } } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 5e036d3d46a7c..326a7ba8c8a69 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -942,8 +942,7 @@ class PhaseIdealLoop : public PhaseTransform { static void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop); #endif private: - static void get_opaque_template_assertion_predicate_nodes(ParsePredicateSuccessProj* parse_predicate_proj, - Unique_Node_List& list); + static void get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false); void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head); void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, CountedLoopNode* remaining_loop_head, @@ -1658,7 +1657,28 @@ class PhaseIdealLoop : public PhaseTransform { _nodes_required = UINT_MAX; } + public: + // Clone Parse Predicates to slow and fast loop when unswitching a loop + void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, + IfProjNode*& true_path_loop_entry, + IfProjNode*& false_path_loop_entry); private: + void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, + const PredicateBlock* predicate_block, + Deoptimization::DeoptReason reason, + IfProjNode*& true_path_loop_entry, + IfProjNode*& false_path_loop_entry); + void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason, + IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred); + IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, + Deoptimization::DeoptReason reason, bool slow_loop); + void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, + ParsePredicateSuccessProj* old_parse_predicate_proj, + ParsePredicateNode* true_path_loop_parse_predicate, + ParsePredicateNode* false_path_loop_parse_predicate); + IfTrueNode* clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj, + ParsePredicateNode* unswitched_loop_parse_predicate); + static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN; bool _created_loop_node; DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);) diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 5be0db2699c39..c944506c547ee 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -82,25 +82,6 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p return nullptr; } -ParsePredicateNode* ParsePredicate::clone_to_unswitched_loop(Node* new_control, const bool is_false_path_loop, - PhaseIdealLoop* phase) const { - ParsePredicateSuccessProj* success_proj = phase->create_new_if_for_predicate(_success_proj, new_control, - _parse_predicate_node->deopt_reason(), - Op_ParsePredicate, is_false_path_loop); - NOT_PRODUCT(trace_cloned_parse_predicate(is_false_path_loop, success_proj)); - return success_proj->in(0)->as_ParsePredicate(); -} - -#ifndef PRODUCT -void ParsePredicate::trace_cloned_parse_predicate(const bool is_false_path_loop, - const ParsePredicateSuccessProj* success_proj) { - if (TraceLoopPredicate) { - tty->print("Parse Predicate cloned to %s path loop: ", is_false_path_loop ? "false" : "true"); - success_proj->in(0)->dump(); - } -} -#endif // NOT PRODUCT - Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) { CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); if (uct_call == nullptr) { @@ -215,13 +196,6 @@ IfTrueNode* TemplateAssertionPredicate::initialize(PhaseIdealLoop* phase, Node* return success_proj; } -// Kills the Template Assertion Predicate by setting the condition to true. Will be folded away in the next IGVN round. -void TemplateAssertionPredicate::kill(PhaseIdealLoop* phase) const { - ConINode* true_con = phase->igvn().intcon(1); - phase->set_ctrl(true_con, phase->C->root()); - phase->igvn().replace_input_of(_if_node, 1, true_con); -} - #ifdef ASSERT // Class to verify Initialized and Template Assertion Predicates by trying to find OpaqueLoop*Nodes. class OpaqueLoopNodesVerifier : public BFSActions { @@ -905,41 +879,6 @@ void Predicates::dump_for_loop(LoopNode* loop_node) { } #endif // NOT PRODUCT -TargetLoopPredicateChain::TargetLoopPredicateChain(LoopNode* loop_head, const NodeInLoopBody& node_in_loop_body, - PhaseIdealLoop* phase) - : _loop_selector_proj(loop_head->in(LoopNode::EntryControl)), - _node_in_loop_body(node_in_loop_body), - _phase(phase) {} - -// Clones the provided Template Assertion Predicate to the head of the current predicate chain at the target loop. -// -// This is done in two steps: -// -// old source loop entry old target loop entry old target loop entry -// | | | | -// Template Assertion 1. clone | Cloned Template 2. rewire Cloned Template -// Predicate 1 ===> | Assertion Predicate ===> Assertion Predicate -// | | | -// source loop head target loop head target loop head -void TargetLoopPredicateChain::clone_template_assertion_predicate_to_chain( - const TemplateAssertionPredicate& template_assertion_predicate) { - Node* current_predicate_chain_head = predicate_chain_head(); - IfTrueNode* cloned_template_success_proj = template_assertion_predicate.clone(_loop_selector_proj, _phase); - template_assertion_predicate.rewire_loop_data_dependencies(cloned_template_success_proj, _node_in_loop_body, - _phase); - rewire_to_target_chain_head(cloned_template_success_proj->in(0)->as_If(), current_predicate_chain_head); -} - -// Rewires the newly cloned Template Assertion Predicates to the head of the current predicate chain at the target loop. -void TargetLoopPredicateChain::rewire_to_target_chain_head(IfNode* cloned_template_assertion_predicate, - Node* current_predicate_chain_head) const { - if (current_predicate_chain_head->is_Loop()) { - _phase->replace_loop_entry(current_predicate_chain_head->as_Loop(), cloned_template_assertion_predicate); - } else { - _phase->replace_control(current_predicate_chain_head, cloned_template_assertion_predicate); - } -} - CreateAssertionPredicatesVisitor::CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head, PhaseIdealLoop* phase, const NodeInLoopBody& node_in_loop_body, @@ -1009,8 +948,8 @@ IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_in // x // | old target // Template Assertion loop entry -// Predicate 1 old target clone | \ -// | loop entry TAP 2 | cloned Template Assertion +// Predicate 1 old target clone | \ +// | loop entry TAP 2 | cloned Template Assertion // Template Assertion | ======> | Predicate 2 // Predicate 2 target loop | // | target loop #_current_predicate_chain_head @@ -1039,42 +978,6 @@ void CreateAssertionPredicatesVisitor::rewire_to_old_predicate_chain_head( } } -CloneUnswitchedLoopPredicatesVisitor::CloneUnswitchedLoopPredicatesVisitor( - LoopNode* true_path_loop_head, LoopNode* false_path_loop_head, - const NodeInOriginalLoopBody& node_in_true_path_loop_body, const NodeInClonedLoopBody& node_in_false_path_loop_body, - PhaseIdealLoop* phase) - : _true_path_loop_predicate_chain(true_path_loop_head, node_in_true_path_loop_body, phase), - _false_path_loop_predicate_chain(false_path_loop_head, node_in_false_path_loop_body, phase), - _phase(phase), - _has_hoisted_check_parse_predicates(false) {} - -// Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found. -// The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block. -void CloneUnswitchedLoopPredicatesVisitor::visit(const ParsePredicate& parse_predicate) { - Deoptimization::DeoptReason deopt_reason = parse_predicate.head()->deopt_reason(); - if (deopt_reason == Deoptimization::Reason_predicate || - deopt_reason == Deoptimization::Reason_profile_predicate) { - _has_hoisted_check_parse_predicates = true; - } - - _true_path_loop_predicate_chain.clone_parse_predicate_to_chain(parse_predicate, true); - _false_path_loop_predicate_chain.clone_parse_predicate_to_chain(parse_predicate, false); - parse_predicate.kill(_phase->igvn()); -} - -// Clone the Template Assertion Predicate, which is currently found before the newly added unswitched loop selector, -// to the true path and false path loop. -void CloneUnswitchedLoopPredicatesVisitor::visit(const TemplateAssertionPredicate& template_assertion_predicate) { - if (!_has_hoisted_check_parse_predicates) { - // Only process if we are in the correct Predicate Block. - return; - } - - _true_path_loop_predicate_chain.clone_template_assertion_predicate_to_chain(template_assertion_predicate); - _false_path_loop_predicate_chain.clone_template_assertion_predicate_to_chain(template_assertion_predicate); - template_assertion_predicate.kill(_phase); -} - // Clone the Template Assertion Predicate and set a new input for the OpaqueLoopStrideNode. void UpdateStrideForAssertionPredicates::visit(const TemplateAssertionPredicate& template_assertion_predicate) { if (!template_assertion_predicate.is_last_value()) { diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index e5b939d0b2f1c..4cd909376aa0d 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -232,7 +232,7 @@ class Predicate : public StackObj { // Generic predicate visitor that does nothing. Subclass this visitor to add customized actions for each predicate. // The visit methods of this visitor are called from the predicate iterator classes which walk the predicate chain. // Use the UnifiedPredicateVisitor if the type of the predicate does not matter. -class PredicateVisitor : public StackObj { +class PredicateVisitor : StackObj { public: virtual void visit(const ParsePredicate& parse_predicate) {} virtual void visit(const RuntimePredicate& runtime_predicate) {} @@ -297,8 +297,6 @@ class ParsePredicate : public Predicate { } static ParsePredicateNode* init_parse_predicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason); - NOT_PRODUCT(static void trace_cloned_parse_predicate(bool is_false_path_loop, - const ParsePredicateSuccessProj* success_proj);) public: ParsePredicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason) @@ -327,14 +325,6 @@ class ParsePredicate : public Predicate { assert(is_valid(), "must be valid"); return _success_proj; } - - ParsePredicateNode* clone_to_unswitched_loop(Node* new_control, bool is_false_path_loop, PhaseIdealLoop* phase) const; - - // Kills this Parse Predicate by marking it useless. Will be folded away in the next IGVN round. - void kill(PhaseIterGVN& igvn) const { - _parse_predicate_node->mark_useless(); - igvn._worklist.push(_parse_predicate_node); - } }; // Class to represent a Runtime Predicate which always has an associated UCT on the failing path. @@ -409,7 +399,6 @@ class TemplateAssertionPredicate : public Predicate { IfTrueNode* initialize(PhaseIdealLoop* phase, Node* new_control) const; void rewire_loop_data_dependencies(IfTrueNode* target_predicate, const NodeInLoopBody& data_in_loop_body, PhaseIdealLoop* phase) const; - void kill(PhaseIdealLoop* phase) const; static bool is_predicate(Node* node); #ifdef ASSERT @@ -989,31 +978,6 @@ class NodeInClonedLoopBody : public NodeInLoopBody { } }; -// This class can be used to create predicates at a target loop where a new chain of predicates need to be established. -class TargetLoopPredicateChain : public StackObj { - Node* const _loop_selector_proj; - const NodeInLoopBody& _node_in_loop_body; - PhaseIdealLoop* const _phase; - - Node* predicate_chain_head() const { - return _loop_selector_proj->unique_ctrl_out(); - } - - void rewire_to_target_chain_head(IfNode* cloned_template_assertion_predicate, Node* current_predicate_chain_head) const; - - public: - TargetLoopPredicateChain(LoopNode* loop_head, const NodeInLoopBody& node_in_loop_body, PhaseIdealLoop* phase); - NONCOPYABLE(TargetLoopPredicateChain); - - // Clones the provided Parse Predicate to the head of the current predicate chain at the target loop. - // This method is used for Loop Unswitching. - void clone_parse_predicate_to_chain(const ParsePredicate& parse_predicate, bool is_true_path_loop) { - parse_predicate.clone_to_unswitched_loop(_loop_selector_proj, !is_true_path_loop, _phase); - } - - void clone_template_assertion_predicate_to_chain(const TemplateAssertionPredicate& template_assertion_predicate); -}; - // Visitor to create Initialized Assertion Predicates at a target loop from Template Assertion Predicates from a source // loop. This visitor can be used in combination with a PredicateIterator. class CreateAssertionPredicatesVisitor : public PredicateVisitor { @@ -1042,43 +1006,25 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor { void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; }; -// Visitor to clone Parse and Template Assertion Predicates from a loop to its unswitched true and false path loop. -// The cloned predicates are not updated in any way. Thus, an Initialized Assertion Predicate is also not required to -// be created. Note that the data dependencies from the Template Assertion Predicates are also updated to the newly -// cloned Templates, depending on whether they belong to the true or false path loop. -class CloneUnswitchedLoopPredicatesVisitor : public PredicateVisitor { - TargetLoopPredicateChain _true_path_loop_predicate_chain; - TargetLoopPredicateChain _false_path_loop_predicate_chain; - - PhaseIdealLoop* const _phase; - bool _has_hoisted_check_parse_predicates; - - public: - CloneUnswitchedLoopPredicatesVisitor(LoopNode* true_path_loop_head, - LoopNode* false_path_loop_head, - const NodeInOriginalLoopBody& node_in_true_path_loop_body, - const NodeInClonedLoopBody& node_in_false_path_loop_body, - PhaseIdealLoop* phase); - NONCOPYABLE(CloneUnswitchedLoopPredicatesVisitor); - - using PredicateVisitor::visit; - - void visit(const ParsePredicate& parse_predicate) override; - void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; -}; - -// This visitor collects all OpaqueTemplateAssertionNodes of Template Assertion Predicates. This is used for cleaning -// up unused Template Assertion Predicates. -class OpaqueTemplateAssertionPredicateCollector : public PredicateVisitor { +// This visitor collects all Template Assertion Predicates If nodes or the corresponding Opaque nodes, depending on the +// provided 'get_opaque' flag, to the provided list. +class TemplateAssertionPredicateCollector : public PredicateVisitor { Unique_Node_List& _list; + const bool _get_opaque; public: - explicit OpaqueTemplateAssertionPredicateCollector(Unique_Node_List& list) : _list(list) {} + TemplateAssertionPredicateCollector(Unique_Node_List& list, const bool get_opaque) + : _list(list), + _get_opaque(get_opaque) {} using PredicateVisitor::visit; void visit(const TemplateAssertionPredicate& template_assertion_predicate) override { - _list.push(template_assertion_predicate.opaque_node()); + if (_get_opaque) { + _list.push(template_assertion_predicate.opaque_node()); + } else { + _list.push(template_assertion_predicate.tail()); + } } };