diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 5096cfb809126..f348db62f2278 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -100,8 +100,7 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, // is an IfTrue projection. This code is also used to clone predicates to cloned loops. IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_success_proj, Node* new_entry, const Deoptimization::DeoptReason reason, - const int opcode, const bool rewire_uncommon_proj_phi_inputs, - AssertionPredicateType assertion_predicate_type) { + const int opcode, const bool rewire_uncommon_proj_phi_inputs) { assert(parse_predicate_success_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!"); ParsePredicateNode* parse_predicate = parse_predicate_success_proj->in(0)->as_ParsePredicate(); ParsePredicateUncommonProj* uncommon_proj = parse_predicate->uncommon_proj(); @@ -143,12 +142,10 @@ IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro IfNode* new_iff = nullptr; switch (opcode) { case Op_If: - new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt - NOT_PRODUCT(COMMA assertion_predicate_type)); + new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt); break; case Op_RangeCheck: - new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt - NOT_PRODUCT(COMMA assertion_predicate_type)); + new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt); break; case Op_ParsePredicate: new_iff = new ParsePredicateNode(entry, reason, &_igvn); diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 26c9e73773159..829c3cb8a18a6 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1351,9 +1351,8 @@ class PhaseIdealLoop : public PhaseTransform { // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted IfTrueNode* create_new_if_for_predicate( - ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode, - bool rewire_uncommon_proj_phi_inputs = false, - AssertionPredicateType assertion_predicate_type = AssertionPredicateType::None); + ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode, + bool rewire_uncommon_proj_phi_inputs = false); private: // Helper functions for create_new_if_for_predicate() diff --git a/src/hotspot/share/opto/opaquenode.cpp b/src/hotspot/share/opto/opaquenode.cpp index 8a8eea51db6d0..94cd84822a1f3 100644 --- a/src/hotspot/share/opto/opaquenode.cpp +++ b/src/hotspot/share/opto/opaquenode.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "opto/connode.hpp" #include "opto/loopnode.hpp" #include "opto/opaquenode.hpp" #include "opto/phaseX.hpp" diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 7afb6a55f6a84..11d0cac66453d 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -82,20 +82,20 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p return nullptr; } -ParsePredicateNode* ParsePredicate::clone_to_unswitched_loop(Node* new_control, const bool is_false_path_loop, - PhaseIdealLoop* phase) const { +ParsePredicateSuccessProj* ParsePredicate::clone_to_unswitched_loop(Node* new_control, const bool is_true_path_loop, + PhaseIdealLoop* phase) const { ParsePredicateSuccessProj* success_proj = phase->create_new_if_for_predicate(_success_proj, new_control, _parse_predicate_node->deopt_reason(), - Op_ParsePredicate, is_false_path_loop); - NOT_PRODUCT(trace_cloned_parse_predicate(is_false_path_loop, success_proj)); - return success_proj->in(0)->as_ParsePredicate(); + Op_ParsePredicate, is_true_path_loop); + NOT_PRODUCT(trace_cloned_parse_predicate(is_true_path_loop, success_proj)); + return success_proj; } #ifndef PRODUCT -void ParsePredicate::trace_cloned_parse_predicate(const bool is_false_path_loop, +void ParsePredicate::trace_cloned_parse_predicate(const bool is_true_path_loop, const ParsePredicateSuccessProj* success_proj) { if (TraceLoopPredicate) { - tty->print("Parse Predicate cloned to %s path loop: ", is_false_path_loop ? "false" : "true"); + tty->print("Parse Predicate cloned to %s path loop: ", is_true_path_loop ? "true" : "false"); success_proj->in(0)->dump(); } } @@ -217,8 +217,7 @@ IfTrueNode* TemplateAssertionPredicate::initialize(PhaseIdealLoop* phase, Node* // Kills the Template Assertion Predicate by setting the condition to true. Will be folded away in the next IGVN round. void TemplateAssertionPredicate::kill(PhaseIdealLoop* phase) const { - ConINode* true_con = phase->igvn().intcon(1); - phase->set_ctrl(true_con, phase->C->root()); + ConINode* true_con = phase->intcon(1); phase->igvn().replace_input_of(_if_node, 1, true_con); } @@ -904,41 +903,36 @@ void Predicates::dump_for_loop(LoopNode* loop_node) { } #endif // NOT PRODUCT -TargetLoopPredicateChain::TargetLoopPredicateChain(LoopNode* loop_head, const NodeInLoopBody& node_in_loop_body, - PhaseIdealLoop* phase) - : _loop_selector_proj(loop_head->in(LoopNode::EntryControl)), - _node_in_loop_body(node_in_loop_body), +TargetLoopPredicateChain::TargetLoopPredicateChain(LoopNode* loop_head, PhaseIdealLoop* phase) + : DEBUG_ONLY(_old_target_loop_entry(loop_head->in(LoopNode::EntryControl)) COMMA) + DEBUG_ONLY(_node_index_before_cloning(phase->C->unique()) COMMA) + _current_predicate_chain_head(loop_head), _phase(phase) {} -// Clones the provided Template Assertion Predicate to the head of the current predicate chain at the target loop. -// -// This is done in two steps: -// -// old source loop entry old target loop entry old target loop entry -// | | | | -// Template Assertion 1. clone | Cloned Template 2. rewire Cloned Template -// Predicate 1 ===> | Assertion Predicate ===> Assertion Predicate -// | | | -// source loop head target loop head target loop head -void TargetLoopPredicateChain::clone_template_assertion_predicate_to_chain( - const TemplateAssertionPredicate& template_assertion_predicate) { - Node* current_predicate_chain_head = predicate_chain_head(); - IfTrueNode* cloned_template_success_proj = template_assertion_predicate.clone(_loop_selector_proj, _phase); - template_assertion_predicate.rewire_loop_data_dependencies(cloned_template_success_proj, _node_in_loop_body, - _phase); - rewire_to_target_chain_head(cloned_template_success_proj->in(0)->as_If(), current_predicate_chain_head); -} - -// Rewires the newly cloned Template Assertion Predicates to the head of the current predicate chain at the target loop. -void TargetLoopPredicateChain::rewire_to_target_chain_head(IfNode* cloned_template_assertion_predicate, - Node* current_predicate_chain_head) const { - if (current_predicate_chain_head->is_Loop()) { - _phase->replace_loop_entry(current_predicate_chain_head->as_Loop(), cloned_template_assertion_predicate); +// Rewires the newly cloned predicate to the head of the current predicate chain at the target loop. +void TargetLoopPredicateChain::insert_predicate(IfTrueNode* predicate_success_proj) { + rewire_to_target_chain_head(predicate_success_proj); + _current_predicate_chain_head = predicate_success_proj->in(0); + assert(predicate_success_proj->_idx >= _node_index_before_cloning, "must be a newly cloned predicate"); + assert(_current_predicate_chain_head->in(0) == _old_target_loop_entry && + _old_target_loop_entry->unique_ctrl_out() == _current_predicate_chain_head , "must be connected now"); +} + +void TargetLoopPredicateChain::rewire_to_target_chain_head(IfTrueNode* template_assertion_predicate_success_proj) const { + if (_current_predicate_chain_head->is_Loop()) { + _phase->replace_loop_entry(_current_predicate_chain_head->as_Loop(), template_assertion_predicate_success_proj); } else { - _phase->replace_control(current_predicate_chain_head, cloned_template_assertion_predicate); + _phase->replace_control(_current_predicate_chain_head, template_assertion_predicate_success_proj); } } +ClonePredicateToTargetLoop::ClonePredicateToTargetLoop(LoopNode* target_loop_head, const NodeInLoopBody& node_in_loop_body, + PhaseIdealLoop* phase) + : _old_target_loop_entry(target_loop_head->in(LoopNode::EntryControl)), + _target_loop_predicate_chain(target_loop_head, phase), + _node_in_loop_body(node_in_loop_body), + _phase(phase) {} + CreateAssertionPredicatesVisitor::CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head, PhaseIdealLoop* phase, const NodeInLoopBody& node_in_loop_body, @@ -1042,8 +1036,8 @@ CloneUnswitchedLoopPredicatesVisitor::CloneUnswitchedLoopPredicatesVisitor( LoopNode* true_path_loop_head, LoopNode* false_path_loop_head, const NodeInOriginalLoopBody& node_in_true_path_loop_body, const NodeInClonedLoopBody& node_in_false_path_loop_body, PhaseIdealLoop* phase) - : _true_path_loop_predicate_chain(true_path_loop_head, node_in_true_path_loop_body, phase), - _false_path_loop_predicate_chain(false_path_loop_head, node_in_false_path_loop_body, phase), + : _clone_predicate_to_true_path_loop(true_path_loop_head, node_in_true_path_loop_body, phase), + _clone_predicate_to_false_path_loop(false_path_loop_head, node_in_false_path_loop_body, phase), _phase(phase), _has_hoisted_check_parse_predicates(false) {} @@ -1056,8 +1050,8 @@ void CloneUnswitchedLoopPredicatesVisitor::visit(const ParsePredicate& parse_pre _has_hoisted_check_parse_predicates = true; } - _true_path_loop_predicate_chain.clone_parse_predicate_to_chain(parse_predicate, true); - _false_path_loop_predicate_chain.clone_parse_predicate_to_chain(parse_predicate, false); + _clone_predicate_to_true_path_loop.clone_parse_predicate(parse_predicate, true); + _clone_predicate_to_false_path_loop.clone_parse_predicate(parse_predicate, false); parse_predicate.kill(_phase->igvn()); } @@ -1069,8 +1063,8 @@ void CloneUnswitchedLoopPredicatesVisitor::visit(const TemplateAssertionPredicat return; } - _true_path_loop_predicate_chain.clone_template_assertion_predicate_to_chain(template_assertion_predicate); - _false_path_loop_predicate_chain.clone_template_assertion_predicate_to_chain(template_assertion_predicate); + _clone_predicate_to_true_path_loop.clone_template_assertion_predicate(template_assertion_predicate); + _clone_predicate_to_false_path_loop.clone_template_assertion_predicate(template_assertion_predicate); template_assertion_predicate.kill(_phase); } diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index c8c338e793f35..bcba191335a4a 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -26,7 +26,6 @@ #define SHARE_OPTO_PREDICATES_HPP #include "opto/cfgnode.hpp" -#include "opto/connode.hpp" #include "opto/opaquenode.hpp" class IdealLoopTree; @@ -297,7 +296,7 @@ class ParsePredicate : public Predicate { } static ParsePredicateNode* init_parse_predicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason); - NOT_PRODUCT(static void trace_cloned_parse_predicate(bool is_false_path_loop, + NOT_PRODUCT(static void trace_cloned_parse_predicate(bool is_true_path_loop, const ParsePredicateSuccessProj* success_proj);) public: @@ -328,10 +327,11 @@ class ParsePredicate : public Predicate { return _success_proj; } - ParsePredicateNode* clone_to_unswitched_loop(Node* new_control, bool is_false_path_loop, PhaseIdealLoop* phase) const; + ParsePredicateSuccessProj* clone_to_unswitched_loop(Node* new_control, bool is_true_path_loop, + PhaseIdealLoop* phase) const; // Kills this Parse Predicate by marking it useless. Will be folded away in the next IGVN round. - void kill(PhaseIterGVN& igvn) const { + void kill(const PhaseIterGVN& igvn) const { _parse_predicate_node->mark_useless(); igvn._worklist.push(_parse_predicate_node); } @@ -989,31 +989,6 @@ class NodeInClonedLoopBody : public NodeInLoopBody { } }; -// This class can be used to create predicates at a target loop where a new chain of predicates need to be established. -class TargetLoopPredicateChain : public StackObj { - Node* const _loop_selector_proj; - const NodeInLoopBody& _node_in_loop_body; - PhaseIdealLoop* const _phase; - - Node* predicate_chain_head() const { - return _loop_selector_proj->unique_ctrl_out(); - } - - void rewire_to_target_chain_head(IfNode* cloned_template_assertion_predicate, Node* current_predicate_chain_head) const; - - public: - TargetLoopPredicateChain(LoopNode* loop_head, const NodeInLoopBody& node_in_loop_body, PhaseIdealLoop* phase); - NONCOPYABLE(TargetLoopPredicateChain); - - // Clones the provided Parse Predicate to the head of the current predicate chain at the target loop. - // This method is used for Loop Unswitching. - void clone_parse_predicate_to_chain(const ParsePredicate& parse_predicate, bool is_true_path_loop) { - parse_predicate.clone_to_unswitched_loop(_loop_selector_proj, !is_true_path_loop, _phase); - } - - void clone_template_assertion_predicate_to_chain(const TemplateAssertionPredicate& template_assertion_predicate); -}; - // Visitor to create Initialized Assertion Predicates at a target loop from Template Assertion Predicates from a source // loop. This visitor can be used in combination with a PredicateIterator. class CreateAssertionPredicatesVisitor : public PredicateVisitor { @@ -1042,13 +1017,72 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor { void visit(const TemplateAssertionPredicate& template_assertion_predicate) override; }; +// This class rewires newly cloned predicates to a target loop predicate chain. +class TargetLoopPredicateChain : public StackObj { + DEBUG_ONLY(const Node* const _old_target_loop_entry;) + DEBUG_ONLY(const node_idx_t _node_index_before_cloning;) + Node* _current_predicate_chain_head; + PhaseIdealLoop* const _phase; + + void rewire_to_target_chain_head(IfTrueNode* template_assertion_predicate_success_proj) const; + +public: + TargetLoopPredicateChain(LoopNode* loop_head, PhaseIdealLoop* phase); + NONCOPYABLE(TargetLoopPredicateChain); + + // Parse Predicates are already correctly rewired with create_new_predicate_if(). Therefore, this method only updates + // the internal bookkeeping to set the newly cloned_parse_predicate as new predicate chain head. + void insert_predicate(IfTrueNode* predicate_success_proj); +}; + +// This class clones Parse and Template Assertion Predicates to the provided target loop. This also involves rewiring +// of any data pinned to Template Assertion Predicates. Each time a predicate is cloned it is inserted at the top of +// previously cloned predicates. This ensures that target loop predicate chain order of the newly cloned predicates is +// the same as in the source loop from which the predicates were cloned from. +// +// Template Assertion Predicate Example: +// +// _old_target_loop_entry _old_target_loop_entry _old_target_loop_entry +// | | | | +// Template Assertion | Cloned Template 2. rewire data Cloned Template +// Predicate 1. clone | Assertion Predicate and predicate Assertion Predicate +// | \ =======> | ===============> | \ +// | data | | data +// | | | +// source loop head target loop head target loop head +class ClonePredicateToTargetLoop : public StackObj { + Node* const _old_target_loop_entry; // Used as control for each newly inserted predicate. + TargetLoopPredicateChain _target_loop_predicate_chain; + const NodeInLoopBody& _node_in_loop_body; + PhaseIdealLoop* const _phase; + +public: + ClonePredicateToTargetLoop(LoopNode* target_loop_head, const NodeInLoopBody& node_in_loop_body, PhaseIdealLoop* phase); + + // Clones the provided Parse Predicate to the head of the current predicate chain at the target loop. + void clone_parse_predicate(const ParsePredicate& parse_predicate, bool is_true_path_loop) { + ParsePredicateSuccessProj* cloned_parse_predicate_success_proj = + parse_predicate.clone_to_unswitched_loop(_old_target_loop_entry, is_true_path_loop, _phase); + _target_loop_predicate_chain.insert_predicate(cloned_parse_predicate_success_proj); + } + + // Clones the provided Template Assertion Predicate to the head of the current predicate chain at the target loop. + void clone_template_assertion_predicate(const TemplateAssertionPredicate& template_assertion_predicate) { + IfTrueNode* cloned_template_success_proj = template_assertion_predicate.clone(_old_target_loop_entry, + _phase); + template_assertion_predicate.rewire_loop_data_dependencies(cloned_template_success_proj, _node_in_loop_body, _phase); + _target_loop_predicate_chain.insert_predicate(cloned_template_success_proj); + } +}; + + // Visitor to clone Parse and Template Assertion Predicates from a loop to its unswitched true and false path loop. // The cloned predicates are not updated in any way. Thus, an Initialized Assertion Predicate is also not required to // be created. Note that the data dependencies from the Template Assertion Predicates are also updated to the newly // cloned Templates, depending on whether they belong to the true or false path loop. class CloneUnswitchedLoopPredicatesVisitor : public PredicateVisitor { - TargetLoopPredicateChain _true_path_loop_predicate_chain; - TargetLoopPredicateChain _false_path_loop_predicate_chain; + ClonePredicateToTargetLoop _clone_predicate_to_true_path_loop; + ClonePredicateToTargetLoop _clone_predicate_to_false_path_loop; PhaseIdealLoop* const _phase; bool _has_hoisted_check_parse_predicates;