Skip to content

8254872: Optimize Rotate on AArch64 #1199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

8254872: Optimize Rotate on AArch64 #1199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 43 additions & 107 deletions src/hotspot/cpu/aarch64/aarch64.ad
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12786,160 +12786,96 @@ instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift,

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE

// rol expander
instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
%{
effect(DEF dst, USE src, USE shift);

format %{ "rol $dst, $src, $shift" %}
ins_cost(INSN_COST * 3);
ins_encode %{
__ subw(rscratch1, zr, as_Register($shift$$reg));
__ rorv(as_Register($dst$$reg), as_Register($src$$reg),
rscratch1);
%}
ins_pipe(ialu_reg_reg_vshift);
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
match(Set dst (RotateRight src shift));

// rol expander
instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
%{
effect(DEF dst, USE src, USE shift);
ins_cost(INSN_COST);
format %{ "ror $dst, $src, $shift" %}

format %{ "rol $dst, $src, $shift" %}
ins_cost(INSN_COST * 3);
ins_encode %{
__ subw(rscratch1, zr, as_Register($shift$$reg));
__ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
rscratch1);
%}
ins_pipe(ialu_reg_reg_vshift);
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
%{
match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));

expand %{
rolL_rReg(dst, src, shift, cr);
%}
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
%{
match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));

expand %{
rolL_rReg(dst, src, shift, cr);
__ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
$shift$$constant & 0x1f);
%}
ins_pipe(ialu_reg_reg_vshift);
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
%{
match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
match(Set dst (RotateRight src shift));

expand %{
rolI_rReg(dst, src, shift, cr);
%}
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
%{
match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
ins_cost(INSN_COST);
format %{ "ror $dst, $src, $shift" %}

expand %{
rolI_rReg(dst, src, shift, cr);
ins_encode %{
__ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
$shift$$constant & 0x3f);
%}
ins_pipe(ialu_reg_reg_vshift);
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE

// ror expander
instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
%{
effect(DEF dst, USE src, USE shift);
match(Set dst (RotateRight src shift));

format %{ "ror $dst, $src, $shift" %}
ins_cost(INSN_COST);
format %{ "ror $dst, $src, $shift" %}

ins_encode %{
__ rorv(as_Register($dst$$reg), as_Register($src$$reg),
as_Register($shift$$reg));
%}
__ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
%}
ins_pipe(ialu_reg_reg_vshift);
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE

// ror expander
instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
%{
effect(DEF dst, USE src, USE shift);
match(Set dst (RotateRight src shift));

format %{ "ror $dst, $src, $shift" %}
ins_cost(INSN_COST);
format %{ "ror $dst, $src, $shift" %}

ins_encode %{
__ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
as_Register($shift$$reg));
%}
__ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
%}
ins_pipe(ialu_reg_reg_vshift);
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
%{
match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
match(Set dst (RotateLeft src shift));

expand %{
rorL_rReg(dst, src, shift, cr);
%}
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
%{
match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
ins_cost(INSN_COST);
format %{ "rol $dst, $src, $shift" %}

expand %{
rorL_rReg(dst, src, shift, cr);
ins_encode %{
__ subw(rscratch1, zr, as_Register($shift$$reg));
__ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
%}
ins_pipe(ialu_reg_reg_vshift);
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
%{
match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
match(Set dst (RotateLeft src shift));

expand %{
rorI_rReg(dst, src, shift, cr);
%}
%}

// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
%{
match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
ins_cost(INSN_COST);
format %{ "rol $dst, $src, $shift" %}

expand %{
rorI_rReg(dst, src, shift, cr);
ins_encode %{
__ subw(rscratch1, zr, as_Register($shift$$reg));
__ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
%}
ins_pipe(ialu_reg_reg_vshift);
%}


Expand Down
81 changes: 19 additions & 62 deletions src/hotspot/cpu/aarch64/aarch64_ad.m4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,75 +329,32 @@ EXTRACT_INSN(L, 63, Or, extr)
EXTRACT_INSN(I, 31, Or, extrw)
EXTRACT_INSN(L, 63, Add, extr)
EXTRACT_INSN(I, 31, Add, extrw)
define(`ROL_EXPAND', `// This pattern is automatically generated from aarch64_ad.m4
define(ROTATE_INSN, `// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE

// $2 expander
instruct $2$1_rReg(iReg$1NoSp dst, iReg$1 src, iRegI shift, rFlagsReg cr)
%{
effect(DEF dst, USE src, USE shift);

format %{ "$2 $dst, $src, $shift" %}
ins_cost(INSN_COST * 3);
ins_encode %{
__ subw(rscratch1, zr, as_Register($shift$$reg));
__ $3(as_Register($dst$$reg), as_Register($src$$reg),
rscratch1);
%}
ins_pipe(ialu_reg_reg_vshift);
%}
')
define(`ROR_EXPAND', `// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE

// $2 expander
instruct $2$1_rReg(iReg$1NoSp dst, iReg$1 src, iRegI shift, rFlagsReg cr)
instruct $2$1_$3(iReg$1NoSp dst, iReg$1 src, ifelse($3, reg, iReg, imm)I shift)
%{
effect(DEF dst, USE src, USE shift);
match(Set dst (ifelse($2, ror, RotateRight, RotateLeft) src shift));

format %{ "$2 $dst, $src, $shift" %}
ins_cost(INSN_COST);
ins_encode %{
__ $3(as_Register($dst$$reg), as_Register($src$$reg),
as_Register($shift$$reg));
%}
ins_pipe(ialu_reg_reg_vshift);
%}
')dnl
define(ROL_INSN, `// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct $3$1_rReg_Var_C$2(iReg$1NoSp dst, iReg$1 src, iRegI shift, immI$2 c$2, rFlagsReg cr)
%{
match(Set dst (Or$1 (LShift$1 src shift) (URShift$1 src (SubI c$2 shift))));

expand %{
$3$1_rReg(dst, src, shift, cr);
%}
%}
')dnl
define(ROR_INSN, `// This pattern is automatically generated from aarch64_ad.m4
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
instruct $3$1_rReg_Var_C$2(iReg$1NoSp dst, iReg$1 src, iRegI shift, immI$2 c$2, rFlagsReg cr)
%{
match(Set dst (Or$1 (URShift$1 src shift) (LShift$1 src (SubI c$2 shift))));

expand %{
$3$1_rReg(dst, src, shift, cr);
format %{ "ifelse($2, ror, ror, rol) $dst, $src, $shift" %}

ifelse($2, rol, ins_encode %{
__ subw(rscratch1, zr, as_Register($shift$$reg));, ins_encode %{)
__ ifelse($3, imm,
ifelse($1, I, extrw, extr)(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
$shift$$constant & ifelse($1, I, 0x1f, 0x3f)),
ifelse($1, I, rorvw, rorv)(as_Register($dst$$reg), as_Register($src$$reg), ifelse($2, rol, rscratch1, as_Register($shift$$reg))));
%}
ins_pipe(ialu_reg_reg_vshift);
%}
')dnl
ROL_EXPAND(L, rol, rorv)
ROL_EXPAND(I, rol, rorvw)
ROL_INSN(L, _64, rol)
ROL_INSN(L, 0, rol)
ROL_INSN(I, _32, rol)
ROL_INSN(I, 0, rol)
ROR_EXPAND(L, ror, rorv)
ROR_EXPAND(I, ror, rorvw)
ROR_INSN(L, _64, ror)
ROR_INSN(L, 0, ror)
ROR_INSN(I, _32, ror)
ROR_INSN(I, 0, ror)
ROTATE_INSN(I, ror, imm)
ROTATE_INSN(L, ror, imm)
ROTATE_INSN(I, ror, reg)
ROTATE_INSN(L, ror, reg)
ROTATE_INSN(I, rol, reg)
ROTATE_INSN(L, rol, reg)
dnl rol_imm has been transformed to ror_imm during GVN.

// Add/subtract (extended)
dnl ADD_SUB_EXTENDED(mode, size, add node, shift node, insn, shift type, wordsize
Expand Down
16 changes: 16 additions & 0 deletions src/hotspot/share/opto/mulnode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1489,6 +1489,22 @@ const Type* RotateLeftNode::Value(PhaseGVN* phase) const {
}
}

Node* RotateLeftNode::Ideal(PhaseGVN *phase, bool can_reshape) {
const Type *t1 = phase->type(in(1));
const Type *t2 = phase->type(in(2));
if (t2->isa_int() && t2->is_int()->is_con()) {
if (t1->isa_int()) {
int lshift = t2->is_int()->get_con() & 31;
return new RotateRightNode(in(1), phase->intcon(32 - (lshift & 31)), TypeInt::INT);
} else {
assert(t1->isa_long(), "Type must be a long");
int lshift = t2->is_int()->get_con() & 63;
return new RotateRightNode(in(1), phase->intcon(64 - (lshift & 63)), TypeLong::LONG);
}
}
return NULL;
}

const Type* RotateRightNode::Value(PhaseGVN* phase) const {
const Type* t1 = phase->type(in(1));
const Type* t2 = phase->type(in(2));
Expand Down
3 changes: 2 additions & 1 deletion src/hotspot/share/opto/mulnode.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -221,6 +221,7 @@ class RotateLeftNode : public TypeNode {
}
virtual int Opcode() const;
virtual const Type* Value(PhaseGVN* phase) const;
virtual Node* Ideal(PhaseGVN* phase, bool can_reshape);
};

//----------------------- RotateRightNode ----------------------------------
Expand Down