Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
@chhagedorn
chhagedorn committed Dec 18, 2024
1 parent 292e1d9 commit f40f90e
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 84 deletions.
9 changes: 3 additions & 6 deletions src/hotspot/share/opto/loopPredicate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,7 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred,
// is an IfTrue projection. This code is also used to clone predicates to cloned loops.
IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_success_proj,
Node* new_entry, const Deoptimization::DeoptReason reason,
const int opcode, const bool rewire_uncommon_proj_phi_inputs,
AssertionPredicateType assertion_predicate_type) {
const int opcode, const bool rewire_uncommon_proj_phi_inputs) {
assert(parse_predicate_success_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
ParsePredicateNode* parse_predicate = parse_predicate_success_proj->in(0)->as_ParsePredicate();
ParsePredicateUncommonProj* uncommon_proj = parse_predicate->uncommon_proj();
Expand Down Expand Up @@ -143,12 +142,10 @@ IfTrueNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessPro
IfNode* new_iff = nullptr;
switch (opcode) {
case Op_If:
new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt
NOT_PRODUCT(COMMA assertion_predicate_type));
new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt);
break;
case Op_RangeCheck:
new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt
NOT_PRODUCT(COMMA assertion_predicate_type));
new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt);
break;
case Op_ParsePredicate:
new_iff = new ParsePredicateNode(entry, reason, &_igvn);
Expand Down
5 changes: 2 additions & 3 deletions src/hotspot/share/opto/loopnode.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1351,9 +1351,8 @@ class PhaseIdealLoop : public PhaseTransform {

// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
IfTrueNode* create_new_if_for_predicate(
ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode,
bool rewire_uncommon_proj_phi_inputs = false,
AssertionPredicateType assertion_predicate_type = AssertionPredicateType::None);
ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, int opcode,
bool rewire_uncommon_proj_phi_inputs = false);

private:
// Helper functions for create_new_if_for_predicate()
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/share/opto/opaquenode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
*/

#include "precompiled.hpp"
#include "opto/connode.hpp"
#include "opto/loopnode.hpp"
#include "opto/opaquenode.hpp"
#include "opto/phaseX.hpp"
Expand Down
82 changes: 38 additions & 44 deletions src/hotspot/share/opto/predicates.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,20 +82,20 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p
return nullptr;
}

ParsePredicateNode* ParsePredicate::clone_to_unswitched_loop(Node* new_control, const bool is_false_path_loop,
PhaseIdealLoop* phase) const {
ParsePredicateSuccessProj* ParsePredicate::clone_to_unswitched_loop(Node* new_control, const bool is_true_path_loop,
PhaseIdealLoop* phase) const {
ParsePredicateSuccessProj* success_proj = phase->create_new_if_for_predicate(_success_proj, new_control,
_parse_predicate_node->deopt_reason(),
Op_ParsePredicate, is_false_path_loop);
NOT_PRODUCT(trace_cloned_parse_predicate(is_false_path_loop, success_proj));
return success_proj->in(0)->as_ParsePredicate();
Op_ParsePredicate, is_true_path_loop);
NOT_PRODUCT(trace_cloned_parse_predicate(is_true_path_loop, success_proj));
return success_proj;
}

#ifndef PRODUCT
void ParsePredicate::trace_cloned_parse_predicate(const bool is_false_path_loop,
void ParsePredicate::trace_cloned_parse_predicate(const bool is_true_path_loop,
const ParsePredicateSuccessProj* success_proj) {
if (TraceLoopPredicate) {
tty->print("Parse Predicate cloned to %s path loop: ", is_false_path_loop ? "false" : "true");
tty->print("Parse Predicate cloned to %s path loop: ", is_true_path_loop ? "true" : "false");
success_proj->in(0)->dump();
}
}
Expand Down Expand Up @@ -217,8 +217,7 @@ IfTrueNode* TemplateAssertionPredicate::initialize(PhaseIdealLoop* phase, Node*

// Kills the Template Assertion Predicate by setting the condition to true. Will be folded away in the next IGVN round.
void TemplateAssertionPredicate::kill(PhaseIdealLoop* phase) const {
ConINode* true_con = phase->igvn().intcon(1);
phase->set_ctrl(true_con, phase->C->root());
ConINode* true_con = phase->intcon(1);
phase->igvn().replace_input_of(_if_node, 1, true_con);
}

Expand Down Expand Up @@ -904,41 +903,36 @@ void Predicates::dump_for_loop(LoopNode* loop_node) {
}
#endif // NOT PRODUCT

TargetLoopPredicateChain::TargetLoopPredicateChain(LoopNode* loop_head, const NodeInLoopBody& node_in_loop_body,
PhaseIdealLoop* phase)
: _loop_selector_proj(loop_head->in(LoopNode::EntryControl)),
_node_in_loop_body(node_in_loop_body),
TargetLoopPredicateChain::TargetLoopPredicateChain(LoopNode* loop_head, PhaseIdealLoop* phase)
: DEBUG_ONLY(_old_target_loop_entry(loop_head->in(LoopNode::EntryControl)) COMMA)
DEBUG_ONLY(_node_index_before_cloning(phase->C->unique()) COMMA)
_current_predicate_chain_head(loop_head),
_phase(phase) {}

// Clones the provided Template Assertion Predicate to the head of the current predicate chain at the target loop.
//
// This is done in two steps:
//
// old source loop entry old target loop entry old target loop entry
// | | | |
// Template Assertion 1. clone | Cloned Template 2. rewire Cloned Template
// Predicate 1 ===> | Assertion Predicate ===> Assertion Predicate
// | | |
// source loop head target loop head target loop head
void TargetLoopPredicateChain::clone_template_assertion_predicate_to_chain(
const TemplateAssertionPredicate& template_assertion_predicate) {
Node* current_predicate_chain_head = predicate_chain_head();
IfTrueNode* cloned_template_success_proj = template_assertion_predicate.clone(_loop_selector_proj, _phase);
template_assertion_predicate.rewire_loop_data_dependencies(cloned_template_success_proj, _node_in_loop_body,
_phase);
rewire_to_target_chain_head(cloned_template_success_proj->in(0)->as_If(), current_predicate_chain_head);
}

// Rewires the newly cloned Template Assertion Predicates to the head of the current predicate chain at the target loop.
void TargetLoopPredicateChain::rewire_to_target_chain_head(IfNode* cloned_template_assertion_predicate,
Node* current_predicate_chain_head) const {
if (current_predicate_chain_head->is_Loop()) {
_phase->replace_loop_entry(current_predicate_chain_head->as_Loop(), cloned_template_assertion_predicate);
// Rewires the newly cloned predicate to the head of the current predicate chain at the target loop.
void TargetLoopPredicateChain::insert_predicate(IfTrueNode* predicate_success_proj) {
rewire_to_target_chain_head(predicate_success_proj);
_current_predicate_chain_head = predicate_success_proj->in(0);
assert(predicate_success_proj->_idx >= _node_index_before_cloning, "must be a newly cloned predicate");
assert(_current_predicate_chain_head->in(0) == _old_target_loop_entry &&
_old_target_loop_entry->unique_ctrl_out() == _current_predicate_chain_head , "must be connected now");
}

void TargetLoopPredicateChain::rewire_to_target_chain_head(IfTrueNode* template_assertion_predicate_success_proj) const {
if (_current_predicate_chain_head->is_Loop()) {
_phase->replace_loop_entry(_current_predicate_chain_head->as_Loop(), template_assertion_predicate_success_proj);
} else {
_phase->replace_control(current_predicate_chain_head, cloned_template_assertion_predicate);
_phase->replace_control(_current_predicate_chain_head, template_assertion_predicate_success_proj);
}
}

ClonePredicateToTargetLoop::ClonePredicateToTargetLoop(LoopNode* target_loop_head, const NodeInLoopBody& node_in_loop_body,
PhaseIdealLoop* phase)
: _old_target_loop_entry(target_loop_head->in(LoopNode::EntryControl)),
_target_loop_predicate_chain(target_loop_head, phase),
_node_in_loop_body(node_in_loop_body),
_phase(phase) {}

CreateAssertionPredicatesVisitor::CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head,
PhaseIdealLoop* phase,
const NodeInLoopBody& node_in_loop_body,
Expand Down Expand Up @@ -1042,8 +1036,8 @@ CloneUnswitchedLoopPredicatesVisitor::CloneUnswitchedLoopPredicatesVisitor(
LoopNode* true_path_loop_head, LoopNode* false_path_loop_head,
const NodeInOriginalLoopBody& node_in_true_path_loop_body, const NodeInClonedLoopBody& node_in_false_path_loop_body,
PhaseIdealLoop* phase)
: _true_path_loop_predicate_chain(true_path_loop_head, node_in_true_path_loop_body, phase),
_false_path_loop_predicate_chain(false_path_loop_head, node_in_false_path_loop_body, phase),
: _clone_predicate_to_true_path_loop(true_path_loop_head, node_in_true_path_loop_body, phase),
_clone_predicate_to_false_path_loop(false_path_loop_head, node_in_false_path_loop_body, phase),
_phase(phase),
_has_hoisted_check_parse_predicates(false) {}

Expand All @@ -1056,8 +1050,8 @@ void CloneUnswitchedLoopPredicatesVisitor::visit(const ParsePredicate& parse_pre
_has_hoisted_check_parse_predicates = true;
}

_true_path_loop_predicate_chain.clone_parse_predicate_to_chain(parse_predicate, true);
_false_path_loop_predicate_chain.clone_parse_predicate_to_chain(parse_predicate, false);
_clone_predicate_to_true_path_loop.clone_parse_predicate(parse_predicate, true);
_clone_predicate_to_false_path_loop.clone_parse_predicate(parse_predicate, false);
parse_predicate.kill(_phase->igvn());
}

Expand All @@ -1069,8 +1063,8 @@ void CloneUnswitchedLoopPredicatesVisitor::visit(const TemplateAssertionPredicat
return;
}

_true_path_loop_predicate_chain.clone_template_assertion_predicate_to_chain(template_assertion_predicate);
_false_path_loop_predicate_chain.clone_template_assertion_predicate_to_chain(template_assertion_predicate);
_clone_predicate_to_true_path_loop.clone_template_assertion_predicate(template_assertion_predicate);
_clone_predicate_to_false_path_loop.clone_template_assertion_predicate(template_assertion_predicate);
template_assertion_predicate.kill(_phase);
}

Expand Down
96 changes: 65 additions & 31 deletions src/hotspot/share/opto/predicates.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
#define SHARE_OPTO_PREDICATES_HPP

#include "opto/cfgnode.hpp"
#include "opto/connode.hpp"
#include "opto/opaquenode.hpp"

class IdealLoopTree;
Expand Down Expand Up @@ -297,7 +296,7 @@ class ParsePredicate : public Predicate {
}

static ParsePredicateNode* init_parse_predicate(Node* parse_predicate_proj, Deoptimization::DeoptReason deopt_reason);
NOT_PRODUCT(static void trace_cloned_parse_predicate(bool is_false_path_loop,
NOT_PRODUCT(static void trace_cloned_parse_predicate(bool is_true_path_loop,
const ParsePredicateSuccessProj* success_proj);)

public:
Expand Down Expand Up @@ -328,10 +327,11 @@ class ParsePredicate : public Predicate {
return _success_proj;
}

ParsePredicateNode* clone_to_unswitched_loop(Node* new_control, bool is_false_path_loop, PhaseIdealLoop* phase) const;
ParsePredicateSuccessProj* clone_to_unswitched_loop(Node* new_control, bool is_true_path_loop,
PhaseIdealLoop* phase) const;

// Kills this Parse Predicate by marking it useless. Will be folded away in the next IGVN round.
void kill(PhaseIterGVN& igvn) const {
void kill(const PhaseIterGVN& igvn) const {
_parse_predicate_node->mark_useless();
igvn._worklist.push(_parse_predicate_node);
}
Expand Down Expand Up @@ -989,31 +989,6 @@ class NodeInClonedLoopBody : public NodeInLoopBody {
}
};

// This class can be used to create predicates at a target loop where a new chain of predicates need to be established.
class TargetLoopPredicateChain : public StackObj {
Node* const _loop_selector_proj;
const NodeInLoopBody& _node_in_loop_body;
PhaseIdealLoop* const _phase;

Node* predicate_chain_head() const {
return _loop_selector_proj->unique_ctrl_out();
}

void rewire_to_target_chain_head(IfNode* cloned_template_assertion_predicate, Node* current_predicate_chain_head) const;

public:
TargetLoopPredicateChain(LoopNode* loop_head, const NodeInLoopBody& node_in_loop_body, PhaseIdealLoop* phase);
NONCOPYABLE(TargetLoopPredicateChain);

// Clones the provided Parse Predicate to the head of the current predicate chain at the target loop.
// This method is used for Loop Unswitching.
void clone_parse_predicate_to_chain(const ParsePredicate& parse_predicate, bool is_true_path_loop) {
parse_predicate.clone_to_unswitched_loop(_loop_selector_proj, !is_true_path_loop, _phase);
}

void clone_template_assertion_predicate_to_chain(const TemplateAssertionPredicate& template_assertion_predicate);
};

// Visitor to create Initialized Assertion Predicates at a target loop from Template Assertion Predicates from a source
// loop. This visitor can be used in combination with a PredicateIterator.
class CreateAssertionPredicatesVisitor : public PredicateVisitor {
Expand Down Expand Up @@ -1042,13 +1017,72 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor {
void visit(const TemplateAssertionPredicate& template_assertion_predicate) override;
};

// This class rewires newly cloned predicates to a target loop predicate chain.
class TargetLoopPredicateChain : public StackObj {
DEBUG_ONLY(const Node* const _old_target_loop_entry;)
DEBUG_ONLY(const node_idx_t _node_index_before_cloning;)
Node* _current_predicate_chain_head;
PhaseIdealLoop* const _phase;

void rewire_to_target_chain_head(IfTrueNode* template_assertion_predicate_success_proj) const;

public:
TargetLoopPredicateChain(LoopNode* loop_head, PhaseIdealLoop* phase);
NONCOPYABLE(TargetLoopPredicateChain);

// Parse Predicates are already correctly rewired with create_new_predicate_if(). Therefore, this method only updates
// the internal bookkeeping to set the newly cloned_parse_predicate as new predicate chain head.
void insert_predicate(IfTrueNode* predicate_success_proj);
};

// This class clones Parse and Template Assertion Predicates to the provided target loop. This also involves rewiring
// of any data pinned to Template Assertion Predicates. Each time a predicate is cloned it is inserted at the top of
// previously cloned predicates. This ensures that target loop predicate chain order of the newly cloned predicates is
// the same as in the source loop from which the predicates were cloned from.
//
// Template Assertion Predicate Example:
//
// _old_target_loop_entry _old_target_loop_entry _old_target_loop_entry
// | | | |
// Template Assertion | Cloned Template 2. rewire data Cloned Template
// Predicate 1. clone | Assertion Predicate and predicate Assertion Predicate
// | \ =======> | ===============> | \
// | data | | data
// | | |
// source loop head target loop head target loop head
class ClonePredicateToTargetLoop : public StackObj {
Node* const _old_target_loop_entry; // Used as control for each newly inserted predicate.
TargetLoopPredicateChain _target_loop_predicate_chain;
const NodeInLoopBody& _node_in_loop_body;
PhaseIdealLoop* const _phase;

public:
ClonePredicateToTargetLoop(LoopNode* target_loop_head, const NodeInLoopBody& node_in_loop_body, PhaseIdealLoop* phase);

// Clones the provided Parse Predicate to the head of the current predicate chain at the target loop.
void clone_parse_predicate(const ParsePredicate& parse_predicate, bool is_true_path_loop) {
ParsePredicateSuccessProj* cloned_parse_predicate_success_proj =
parse_predicate.clone_to_unswitched_loop(_old_target_loop_entry, is_true_path_loop, _phase);
_target_loop_predicate_chain.insert_predicate(cloned_parse_predicate_success_proj);
}

// Clones the provided Template Assertion Predicate to the head of the current predicate chain at the target loop.
void clone_template_assertion_predicate(const TemplateAssertionPredicate& template_assertion_predicate) {
IfTrueNode* cloned_template_success_proj = template_assertion_predicate.clone(_old_target_loop_entry,
_phase);
template_assertion_predicate.rewire_loop_data_dependencies(cloned_template_success_proj, _node_in_loop_body, _phase);
_target_loop_predicate_chain.insert_predicate(cloned_template_success_proj);
}
};


// Visitor to clone Parse and Template Assertion Predicates from a loop to its unswitched true and false path loop.
// The cloned predicates are not updated in any way. Thus, an Initialized Assertion Predicate is also not required to
// be created. Note that the data dependencies from the Template Assertion Predicates are also updated to the newly
// cloned Templates, depending on whether they belong to the true or false path loop.
class CloneUnswitchedLoopPredicatesVisitor : public PredicateVisitor {
TargetLoopPredicateChain _true_path_loop_predicate_chain;
TargetLoopPredicateChain _false_path_loop_predicate_chain;
ClonePredicateToTargetLoop _clone_predicate_to_true_path_loop;
ClonePredicateToTargetLoop _clone_predicate_to_false_path_loop;

PhaseIdealLoop* const _phase;
bool _has_hoisted_check_parse_predicates;
Expand Down

0 comments on commit f40f90e

Please sign in to comment.