Skip to content

Commit 30a2ad5

Browse files
Eric Liunick-arm
Eric Liu
authored and
nick-arm
committed
8254872: Optimize Rotate on AArch64
Reviewed-by: aph, kvn
1 parent eabf3ba commit 30a2ad5

File tree

4 files changed

+80
-170
lines changed

4 files changed

+80
-170
lines changed

‎src/hotspot/cpu/aarch64/aarch64.ad‎

Lines changed: 43 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -12757,160 +12757,96 @@ instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift,
1275712757

1275812758
// This pattern is automatically generated from aarch64_ad.m4
1275912759
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12760-
12761-
// rol expander
12762-
instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12760+
instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
1276312761
%{
12764-
effect(DEF dst, USE src, USE shift);
12765-
12766-
format %{ "rol $dst, $src, $shift" %}
12767-
ins_cost(INSN_COST * 3);
12768-
ins_encode %{
12769-
__ subw(rscratch1, zr, as_Register($shift$$reg));
12770-
__ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12771-
rscratch1);
12772-
%}
12773-
ins_pipe(ialu_reg_reg_vshift);
12774-
%}
12775-
12776-
// This pattern is automatically generated from aarch64_ad.m4
12777-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12762+
match(Set dst (RotateRight src shift));
1277812763

12779-
// rol expander
12780-
instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12781-
%{
12782-
effect(DEF dst, USE src, USE shift);
12764+
ins_cost(INSN_COST);
12765+
format %{ "ror $dst, $src, $shift" %}
1278312766

12784-
format %{ "rol $dst, $src, $shift" %}
12785-
ins_cost(INSN_COST * 3);
1278612767
ins_encode %{
12787-
__ subw(rscratch1, zr, as_Register($shift$$reg));
12788-
__ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12789-
rscratch1);
12790-
%}
12791-
ins_pipe(ialu_reg_reg_vshift);
12792-
%}
12793-
12794-
// This pattern is automatically generated from aarch64_ad.m4
12795-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12796-
instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12797-
%{
12798-
match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
12799-
12800-
expand %{
12801-
rolL_rReg(dst, src, shift, cr);
12802-
%}
12803-
%}
12804-
12805-
// This pattern is automatically generated from aarch64_ad.m4
12806-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807-
instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12808-
%{
12809-
match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
12810-
12811-
expand %{
12812-
rolL_rReg(dst, src, shift, cr);
12768+
__ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12769+
$shift$$constant & 0x1f);
1281312770
%}
12771+
ins_pipe(ialu_reg_reg_vshift);
1281412772
%}
1281512773

1281612774
// This pattern is automatically generated from aarch64_ad.m4
1281712775
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12818-
instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12776+
instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
1281912777
%{
12820-
match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
12778+
match(Set dst (RotateRight src shift));
1282112779

12822-
expand %{
12823-
rolI_rReg(dst, src, shift, cr);
12824-
%}
12825-
%}
12826-
12827-
// This pattern is automatically generated from aarch64_ad.m4
12828-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12829-
instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12830-
%{
12831-
match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
12780+
ins_cost(INSN_COST);
12781+
format %{ "ror $dst, $src, $shift" %}
1283212782

12833-
expand %{
12834-
rolI_rReg(dst, src, shift, cr);
12783+
ins_encode %{
12784+
__ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12785+
$shift$$constant & 0x3f);
1283512786
%}
12787+
ins_pipe(ialu_reg_reg_vshift);
1283612788
%}
1283712789

1283812790
// This pattern is automatically generated from aarch64_ad.m4
1283912791
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840-
12841-
// ror expander
12842-
instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
12792+
instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
1284312793
%{
12844-
effect(DEF dst, USE src, USE shift);
12794+
match(Set dst (RotateRight src shift));
1284512795

12846-
format %{ "ror $dst, $src, $shift" %}
1284712796
ins_cost(INSN_COST);
12797+
format %{ "ror $dst, $src, $shift" %}
12798+
1284812799
ins_encode %{
12849-
__ rorv(as_Register($dst$$reg), as_Register($src$$reg),
12850-
as_Register($shift$$reg));
12851-
%}
12800+
__ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12801+
%}
1285212802
ins_pipe(ialu_reg_reg_vshift);
1285312803
%}
1285412804

1285512805
// This pattern is automatically generated from aarch64_ad.m4
1285612806
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12857-
12858-
// ror expander
12859-
instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
12807+
instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
1286012808
%{
12861-
effect(DEF dst, USE src, USE shift);
12809+
match(Set dst (RotateRight src shift));
1286212810

12863-
format %{ "ror $dst, $src, $shift" %}
1286412811
ins_cost(INSN_COST);
12812+
format %{ "ror $dst, $src, $shift" %}
12813+
1286512814
ins_encode %{
12866-
__ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
12867-
as_Register($shift$$reg));
12868-
%}
12815+
__ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12816+
%}
1286912817
ins_pipe(ialu_reg_reg_vshift);
1287012818
%}
1287112819

1287212820
// This pattern is automatically generated from aarch64_ad.m4
1287312821
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12874-
instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
12822+
instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
1287512823
%{
12876-
match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
12824+
match(Set dst (RotateLeft src shift));
1287712825

12878-
expand %{
12879-
rorL_rReg(dst, src, shift, cr);
12880-
%}
12881-
%}
12882-
12883-
// This pattern is automatically generated from aarch64_ad.m4
12884-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12885-
instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
12886-
%{
12887-
match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
12826+
ins_cost(INSN_COST);
12827+
format %{ "rol $dst, $src, $shift" %}
1288812828

12889-
expand %{
12890-
rorL_rReg(dst, src, shift, cr);
12829+
ins_encode %{
12830+
__ subw(rscratch1, zr, as_Register($shift$$reg));
12831+
__ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
1289112832
%}
12833+
ins_pipe(ialu_reg_reg_vshift);
1289212834
%}
1289312835

1289412836
// This pattern is automatically generated from aarch64_ad.m4
1289512837
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12896-
instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
12838+
instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
1289712839
%{
12898-
match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
12840+
match(Set dst (RotateLeft src shift));
1289912841

12900-
expand %{
12901-
rorI_rReg(dst, src, shift, cr);
12902-
%}
12903-
%}
12904-
12905-
// This pattern is automatically generated from aarch64_ad.m4
12906-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12907-
instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
12908-
%{
12909-
match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
12842+
ins_cost(INSN_COST);
12843+
format %{ "rol $dst, $src, $shift" %}
1291012844

12911-
expand %{
12912-
rorI_rReg(dst, src, shift, cr);
12845+
ins_encode %{
12846+
__ subw(rscratch1, zr, as_Register($shift$$reg));
12847+
__ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
1291312848
%}
12849+
ins_pipe(ialu_reg_reg_vshift);
1291412850
%}
1291512851

1291612852

‎src/hotspot/cpu/aarch64/aarch64_ad.m4‎

Lines changed: 19 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -329,75 +329,32 @@ EXTRACT_INSN(L, 63, Or, extr)
329329
EXTRACT_INSN(I, 31, Or, extrw)
330330
EXTRACT_INSN(L, 63, Add, extr)
331331
EXTRACT_INSN(I, 31, Add, extrw)
332-
define(`ROL_EXPAND', `// This pattern is automatically generated from aarch64_ad.m4
332+
define(ROTATE_INSN, `// This pattern is automatically generated from aarch64_ad.m4
333333
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
334-
335-
// $2 expander
336-
instruct $2$1_rReg(iReg$1NoSp dst, iReg$1 src, iRegI shift, rFlagsReg cr)
337-
%{
338-
effect(DEF dst, USE src, USE shift);
339-
340-
format %{ "$2 $dst, $src, $shift" %}
341-
ins_cost(INSN_COST * 3);
342-
ins_encode %{
343-
__ subw(rscratch1, zr, as_Register($shift$$reg));
344-
__ $3(as_Register($dst$$reg), as_Register($src$$reg),
345-
rscratch1);
346-
%}
347-
ins_pipe(ialu_reg_reg_vshift);
348-
%}
349-
')
350-
define(`ROR_EXPAND', `// This pattern is automatically generated from aarch64_ad.m4
351-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
352-
353-
// $2 expander
354-
instruct $2$1_rReg(iReg$1NoSp dst, iReg$1 src, iRegI shift, rFlagsReg cr)
334+
instruct $2$1_$3(iReg$1NoSp dst, iReg$1 src, ifelse($3, reg, iReg, imm)I shift)
355335
%{
356-
effect(DEF dst, USE src, USE shift);
336+
match(Set dst (ifelse($2, ror, RotateRight, RotateLeft) src shift));
357337

358-
format %{ "$2 $dst, $src, $shift" %}
359338
ins_cost(INSN_COST);
360-
ins_encode %{
361-
__ $3(as_Register($dst$$reg), as_Register($src$$reg),
362-
as_Register($shift$$reg));
363-
%}
364-
ins_pipe(ialu_reg_reg_vshift);
365-
%}
366-
')dnl
367-
define(ROL_INSN, `// This pattern is automatically generated from aarch64_ad.m4
368-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
369-
instruct $3$1_rReg_Var_C$2(iReg$1NoSp dst, iReg$1 src, iRegI shift, immI$2 c$2, rFlagsReg cr)
370-
%{
371-
match(Set dst (Or$1 (LShift$1 src shift) (URShift$1 src (SubI c$2 shift))));
372-
373-
expand %{
374-
$3$1_rReg(dst, src, shift, cr);
375-
%}
376-
%}
377-
')dnl
378-
define(ROR_INSN, `// This pattern is automatically generated from aarch64_ad.m4
379-
// DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
380-
instruct $3$1_rReg_Var_C$2(iReg$1NoSp dst, iReg$1 src, iRegI shift, immI$2 c$2, rFlagsReg cr)
381-
%{
382-
match(Set dst (Or$1 (URShift$1 src shift) (LShift$1 src (SubI c$2 shift))));
383-
384-
expand %{
385-
$3$1_rReg(dst, src, shift, cr);
339+
format %{ "ifelse($2, ror, ror, rol) $dst, $src, $shift" %}
340+
341+
ifelse($2, rol, ins_encode %{
342+
__ subw(rscratch1, zr, as_Register($shift$$reg));, ins_encode %{)
343+
__ ifelse($3, imm,
344+
ifelse($1, I, extrw, extr)(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
345+
$shift$$constant & ifelse($1, I, 0x1f, 0x3f)),
346+
ifelse($1, I, rorvw, rorv)(as_Register($dst$$reg), as_Register($src$$reg), ifelse($2, rol, rscratch1, as_Register($shift$$reg))));
386347
%}
348+
ins_pipe(ialu_reg_reg_vshift);
387349
%}
388350
')dnl
389-
ROL_EXPAND(L, rol, rorv)
390-
ROL_EXPAND(I, rol, rorvw)
391-
ROL_INSN(L, _64, rol)
392-
ROL_INSN(L, 0, rol)
393-
ROL_INSN(I, _32, rol)
394-
ROL_INSN(I, 0, rol)
395-
ROR_EXPAND(L, ror, rorv)
396-
ROR_EXPAND(I, ror, rorvw)
397-
ROR_INSN(L, _64, ror)
398-
ROR_INSN(L, 0, ror)
399-
ROR_INSN(I, _32, ror)
400-
ROR_INSN(I, 0, ror)
351+
ROTATE_INSN(I, ror, imm)
352+
ROTATE_INSN(L, ror, imm)
353+
ROTATE_INSN(I, ror, reg)
354+
ROTATE_INSN(L, ror, reg)
355+
ROTATE_INSN(I, rol, reg)
356+
ROTATE_INSN(L, rol, reg)
357+
dnl rol_imm has been transformed to ror_imm during GVN.
401358

402359
// Add/subtract (extended)
403360
dnl ADD_SUB_EXTENDED(mode, size, add node, shift node, insn, shift type, wordsize

‎src/hotspot/share/opto/mulnode.cpp‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1489,6 +1489,22 @@ const Type* RotateLeftNode::Value(PhaseGVN* phase) const {
14891489
}
14901490
}
14911491

1492+
Node* RotateLeftNode::Ideal(PhaseGVN *phase, bool can_reshape) {
1493+
const Type *t1 = phase->type(in(1));
1494+
const Type *t2 = phase->type(in(2));
1495+
if (t2->isa_int() && t2->is_int()->is_con()) {
1496+
if (t1->isa_int()) {
1497+
int lshift = t2->is_int()->get_con() & 31;
1498+
return new RotateRightNode(in(1), phase->intcon(32 - (lshift & 31)), TypeInt::INT);
1499+
} else {
1500+
assert(t1->isa_long(), "Type must be a long");
1501+
int lshift = t2->is_int()->get_con() & 63;
1502+
return new RotateRightNode(in(1), phase->intcon(64 - (lshift & 63)), TypeLong::LONG);
1503+
}
1504+
}
1505+
return NULL;
1506+
}
1507+
14921508
const Type* RotateRightNode::Value(PhaseGVN* phase) const {
14931509
const Type* t1 = phase->type(in(1));
14941510
const Type* t2 = phase->type(in(2));

‎src/hotspot/share/opto/mulnode.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -221,6 +221,7 @@ class RotateLeftNode : public TypeNode {
221221
}
222222
virtual int Opcode() const;
223223
virtual const Type* Value(PhaseGVN* phase) const;
224+
virtual Node* Ideal(PhaseGVN* phase, bool can_reshape);
224225
};
225226

226227
//----------------------- RotateRightNode ----------------------------------

0 commit comments

Comments
 (0)