Skip to content

8373344: Add support for min/max reduction operations for Float16 #28828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yiwu0b11 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

8373344: Add support for min/max reduction operations for Float16 #28828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions src/hotspot/cpu/aarch64/aarch64_vector.ad
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,16 @@ source %{
return false;
}
break;
case Op_MinReductionVHF:
case Op_MaxReductionVHF:
// Reductions with less than 8 bytes vector length are not supported.
// FEAT_FP16 is enabled if both "fphp" and "asimdhp" features are supported.
// Only the Neon instructions need this check. SVE supports half-precision floats
// by default.
if (length_in_bytes < 8 || (UseSVE == 0 && !is_feat_fp16_supported())) {
return false;
}
break;
case Op_MulReductionVD:
case Op_MulReductionVF:
case Op_MulReductionVI:
Expand Down Expand Up @@ -367,6 +377,8 @@ source %{
case Op_AndReductionV:
case Op_OrReductionV:
case Op_XorReductionV:
case Op_MinReductionVHF:
case Op_MaxReductionVHF:
// Mask is needed for partial Op_VectorMaskFirstTrue, because when the
// input predicate is all-false, the result should be the vector length
// instead of the vector register size.
Expand Down Expand Up @@ -3933,6 +3945,26 @@ instruct reduce_maxL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
ins_pipe(pipe_slow);
%}

// reduction maxHF

instruct reduce_maxHF(vRegF dst, vRegF fsrc, vReg vsrc) %{
match(Set dst (MaxReductionVHF fsrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_maxHF $dst, $fsrc, $vsrc" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ fmaxv($dst$$FloatRegister, get_arrangement(in(2)), $vsrc$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_fmaxv($dst$$FloatRegister, __ H, ptrue, $vsrc$$FloatRegister);
}
__ fmaxh($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}

// reduction maxF

instruct reduce_maxF(vRegF dst, vRegF fsrc, vReg vsrc) %{
Expand Down Expand Up @@ -4014,6 +4046,18 @@ instruct reduce_maxL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg,
ins_pipe(pipe_slow);
%}

instruct reduce_maxHF_masked(vRegF dst, vRegF fsrc, vReg vsrc, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst (MaxReductionVHF (Binary fsrc vsrc) pg));
effect(TEMP_DEF dst);
format %{ "reduce_maxHF_masked $dst, $fsrc, $pg, $vsrc" %}
ins_encode %{
__ sve_fmaxv($dst$$FloatRegister, __ H, $pg$$PRegister, $vsrc$$FloatRegister);
__ fmaxh($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}

instruct reduce_maxF_masked(vRegF dst, vRegF fsrc, vReg vsrc, pRegGov pg) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_FLOAT);
match(Set dst (MaxReductionV (Binary fsrc vsrc) pg));
Expand Down Expand Up @@ -4113,6 +4157,26 @@ instruct reduce_minL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
ins_pipe(pipe_slow);
%}

// reduction minHF

instruct reduce_minHF(vRegF dst, vRegF fsrc, vReg vsrc) %{
match(Set dst (MinReductionVHF fsrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_minHF $dst, $fsrc, $vsrc" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ fminv($dst$$FloatRegister, get_arrangement(in(2)), $vsrc$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_fminv($dst$$FloatRegister, __ H, ptrue, $vsrc$$FloatRegister);
}
__ fminh($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}

// reduction minF

instruct reduce_minF(vRegF dst, vRegF fsrc, vReg vsrc) %{
Expand Down Expand Up @@ -4194,6 +4258,18 @@ instruct reduce_minL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg,
ins_pipe(pipe_slow);
%}

instruct reduce_minHF_masked(vRegF dst, vRegF fsrc, vReg vsrc, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst (MinReductionVHF (Binary fsrc vsrc) pg));
effect(TEMP_DEF dst);
format %{ "reduce_minHF_masked $dst, $fsrc, $pg, $vsrc" %}
ins_encode %{
__ sve_fminv($dst$$FloatRegister, __ H, $pg$$PRegister, $vsrc$$FloatRegister);
__ fminh($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}

instruct reduce_minF_masked(vRegF dst, vRegF fsrc, vReg vsrc, pRegGov pg) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_FLOAT);
match(Set dst (MinReductionV (Binary fsrc vsrc) pg));
Expand Down
87 changes: 45 additions & 42 deletions src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,16 @@ source %{
return false;
}
break;
case Op_MinReductionVHF:
case Op_MaxReductionVHF:
// Reductions with less than 8 bytes vector length are not supported.
// FEAT_FP16 is enabled if both "fphp" and "asimdhp" features are supported.
// Only the Neon instructions need this check. SVE supports half-precision floats
// by default.
if (length_in_bytes < 8 || (UseSVE == 0 && !is_feat_fp16_supported())) {
return false;
}
break;
case Op_MulReductionVD:
case Op_MulReductionVF:
case Op_MulReductionVI:
Expand Down Expand Up @@ -357,6 +367,8 @@ source %{
case Op_AndReductionV:
case Op_OrReductionV:
case Op_XorReductionV:
case Op_MinReductionVHF:
case Op_MaxReductionVHF:
// Mask is needed for partial Op_VectorMaskFirstTrue, because when the
// input predicate is all-false, the result should be the vector length
// instead of the vector register size.
Expand Down Expand Up @@ -2375,50 +2387,31 @@ instruct reduce_$1L_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_F($1, $2, $3, $4, $5, $6 )
dnl REDUCE_MAXMIN_F(type, op_name, insn1, insn2, insn3, insn4)
define(`REDUCE_MAXMIN_F', `
instruct reduce_$1F(vRegF dst, vRegF fsrc, vReg vsrc) %{
predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
match(Set dst ($2 fsrc vsrc));
dnl REDUCE_MAXMIN_FP($1, $2, $3, $4, $5, $6, $7, $8 )
dnl REDUCE_MAXMIN_FP(type, op_name, kind, arg, insn1, insn2, insn3, insn4)
define(`REDUCE_MAXMIN_FP', `
instruct reduce_$1$3(vReg`'ifelse($3, D, D, F) dst, vReg`'ifelse($3, D, D, F) $4, vReg vsrc) %{
ifelse($3, HF, `dnl',
` predicate(Matcher::vector_element_basic_type(n->in(2)) == ifelse($3, F, T_FLOAT, $3, D, T_DOUBLE));')
match(Set dst ($2 $4 vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_$1F $dst, $fsrc, $vsrc" %}
format %{ "reduce_$1$3 $dst, $$4, $vsrc" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
if (length_in_bytes == 8) {
__ $3($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
ifelse($3, HF, `__ $5($dst$$FloatRegister, get_arrangement(in(2)), $vsrc$$FloatRegister);',
$3, F, `if (length_in_bytes == 8) {
__ $5($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
} else {
__ $4($dst$$FloatRegister, __ T4S, $vsrc$$FloatRegister);
}
__ $6($dst$$FloatRegister, __ T4S, $vsrc$$FloatRegister);
}',
$3, D, `__ $5($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);')
} else {
assert(UseSVE > 0, "must be sve");
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ $5($dst$$FloatRegister, __ S, ptrue, $vsrc$$FloatRegister);
__ $7($dst$$FloatRegister, __ ifelse($3, F, S, $3, D, D, $3, HF, H), ptrue, $vsrc$$FloatRegister);
}
__ $6($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REDUCE_MAXMIN_D($1, $2, $3, $4, $5 )
dnl REDUCE_MAXMIN_D(type, op_name, insn1, insn2, insn3)
define(`REDUCE_MAXMIN_D', `
instruct reduce_$1D(vRegD dst, vRegD dsrc, vReg vsrc) %{
predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
match(Set dst ($2 dsrc vsrc));
effect(TEMP_DEF dst);
format %{ "reduce_$1D $dst, $dsrc, $vsrc" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ $3($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
} else {
assert(UseSVE > 0, "must be sve");
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ $4($dst$$FloatRegister, __ D, ptrue, $vsrc$$FloatRegister);
}
__ $5($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
__ $8($dst$$FloatRegister, $dst$$FloatRegister, $$4$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
Expand Down Expand Up @@ -2449,13 +2442,15 @@ dnl
dnl REDUCE_MAXMIN_FP_PREDICATE($1, $2, $3, $4, $5, $6 )
dnl REDUCE_MAXMIN_FP_PREDICATE(type, is_float, arg_name, op_name, insn1, insn2)
define(`REDUCE_MAXMIN_FP_PREDICATE', `
instruct reduce_$1$2_masked(vReg$2 dst, vReg$2 $3, vReg vsrc, pRegGov pg) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == ifelse($2, F, T_FLOAT, T_DOUBLE));
instruct reduce_$1$2_masked(vReg`'ifelse($2, HF, F, $2) dst, vReg`'ifelse($2, HF, F, $2) $3, vReg vsrc, pRegGov pg) %{
predicate(ifelse($2, HF,
`UseSVE > 0',
`UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == ifelse($2, F, T_FLOAT, $2, D, T_DOUBLE)'));
match(Set dst ($4 (Binary $3 vsrc) pg));
effect(TEMP_DEF dst);
format %{ "reduce_$1$2_masked $dst, $$3, $pg, $vsrc" %}
ins_encode %{
__ $5($dst$$FloatRegister, __ ifelse($2, F, S, D), $pg$$PRegister, $vsrc$$FloatRegister);
__ $5($dst$$FloatRegister, __ ifelse($2, F, S, $2, D, D, $2, HF, H), $pg$$PRegister, $vsrc$$FloatRegister);
__ $6($dst$$FloatRegister, $dst$$FloatRegister, $$3$$FloatRegister);
%}
ins_pipe(pipe_slow);
Expand All @@ -2471,15 +2466,19 @@ REDUCE_MAXMIN_I_SVE(max, MaxReductionV)
REDUCE_MAXMIN_L_NEON(max, MaxReductionV)
REDUCE_MAXMIN_L_SVE(max, MaxReductionV)

// reduction maxHF
REDUCE_MAXMIN_FP(max, MaxReductionVHF, HF, fsrc, fmaxv, fmaxv, sve_fmaxv, fmaxh)

// reduction maxF
REDUCE_MAXMIN_F(max, MaxReductionV, fmaxp, fmaxv, sve_fmaxv, fmaxs)
REDUCE_MAXMIN_FP(max, MaxReductionV, F, fsrc, fmaxp, fmaxv, sve_fmaxv, fmaxs)

// reduction maxD
REDUCE_MAXMIN_D(max, MaxReductionV, fmaxp, sve_fmaxv, fmaxd)
REDUCE_MAXMIN_FP(max, MaxReductionV, D, dsrc, fmaxp, fmaxp, sve_fmaxv, fmaxd)

// reduction max - predicated
REDUCE_MAXMIN_INT_PREDICATE(max, I, iRegIorL2I, MaxReductionV)
REDUCE_MAXMIN_INT_PREDICATE(max, L, iRegL, MaxReductionV)
REDUCE_MAXMIN_FP_PREDICATE(max, HF, fsrc, MaxReductionVHF, sve_fmaxv, fmaxh)
REDUCE_MAXMIN_FP_PREDICATE(max, F, fsrc, MaxReductionV, sve_fmaxv, fmaxs)
REDUCE_MAXMIN_FP_PREDICATE(max, D, dsrc, MaxReductionV, sve_fmaxv, fmaxd)

Expand All @@ -2493,15 +2492,19 @@ REDUCE_MAXMIN_I_SVE(min, MinReductionV)
REDUCE_MAXMIN_L_NEON(min, MinReductionV)
REDUCE_MAXMIN_L_SVE(min, MinReductionV)

// reduction minHF
REDUCE_MAXMIN_FP(min, MinReductionVHF, HF, fsrc, fminv, fminv, sve_fminv, fminh)

// reduction minF
REDUCE_MAXMIN_F(min, MinReductionV, fminp, fminv, sve_fminv, fmins)
REDUCE_MAXMIN_FP(min, MinReductionV, F, fsrc, fminp, fminv, sve_fminv, fmins)

// reduction minD
REDUCE_MAXMIN_D(min, MinReductionV, fminp, sve_fminv, fmind)
REDUCE_MAXMIN_FP(min, MinReductionV, D, dsrc, fminp, fminp, sve_fminv, fmind)

// reduction min - predicated
REDUCE_MAXMIN_INT_PREDICATE(min, I, iRegIorL2I, MinReductionV)
REDUCE_MAXMIN_INT_PREDICATE(min, L, iRegL, MinReductionV)
REDUCE_MAXMIN_FP_PREDICATE(min, HF, fsrc, MinReductionVHF, sve_fminv, fminh)
REDUCE_MAXMIN_FP_PREDICATE(min, F, fsrc, MinReductionV, sve_fminv, fmins)
REDUCE_MAXMIN_FP_PREDICATE(min, D, dsrc, MinReductionV, sve_fminv, fmind)

Expand Down
13 changes: 10 additions & 3 deletions src/hotspot/cpu/aarch64/assembler_aarch64.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2695,9 +2695,16 @@ template<typename R, typename... Rx>
#define INSN(NAME, opc) \
void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) { \
starti; \
assert(T == T4S, "arrangement must be T4S"); \
f(0, 31), f((int)T & 1, 30), f(0b101110, 29, 24), f(opc, 23), \
f(T == T4S ? 0 : 1, 22), f(0b110000111110, 21, 10); rf(Vn, 5), rf(Vd, 0); \
assert(T == T4H || T == T8H || T == T4S, "arrangement must be T4H, T8H, or T4S"); \
int op29; \
switch (T) { \
case T4H: op29 = 0b00; break; \
case T8H: op29 = 0b10; break; \
case T4S: op29 = 0b11; break; \
default: ShouldNotReachHere(); \
} \
f(0, 31), f(op29, 30, 29), f(0b01110, 28, 24), f(opc, 23), \
f(0b0110000111110, 22, 10); rf(Vn, 5), rf(Vd, 0); \
}

INSN(fmaxv, 0);
Expand Down
4 changes: 3 additions & 1 deletion src/hotspot/share/adlc/formssel.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4240,7 +4240,9 @@ int MatchRule::is_expensive() const {
strcmp(opType,"MulReductionVF")==0 ||
strcmp(opType,"MulReductionVD")==0 ||
strcmp(opType,"MinReductionV")==0 ||
strcmp(opType,"MinReductionVHF")==0 ||
strcmp(opType,"MaxReductionV")==0 ||
strcmp(opType,"MaxReductionVHF")==0 ||
strcmp(opType,"AndReductionV")==0 ||
strcmp(opType,"OrReductionV")==0 ||
strcmp(opType,"XorReductionV")==0 ||
Expand Down Expand Up @@ -4349,7 +4351,7 @@ bool MatchRule::is_vector() const {
"AddReductionVF", "AddReductionVD",
"MulReductionVI", "MulReductionVL",
"MulReductionVF", "MulReductionVD",
"MaxReductionV", "MinReductionV",
"MaxReductionV", "MaxReductionVHF", "MinReductionV", "MinReductionVHF",
"AndReductionV", "OrReductionV", "XorReductionV",
"MulAddVS2VI", "MacroLogicV",
"LShiftCntV","RShiftCntV",
Expand Down
2 changes: 2 additions & 0 deletions src/hotspot/share/opto/classes.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,9 @@ macro(MaxVHF)
macro(UMinV)
macro(UMaxV)
macro(MinReductionV)
macro(MinReductionVHF)
macro(MaxReductionV)
macro(MaxReductionVHF)
macro(CompressV)
macro(CompressM)
macro(ExpandV)
Expand Down
6 changes: 4 additions & 2 deletions src/hotspot/share/opto/compile.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3189,8 +3189,8 @@ void Compile::final_graph_reshaping_impl(Node *n, Final_Reshape_Counts& frc, Uni
// Check for commutative opcode
switch( nop ) {
case Op_AddI: case Op_AddF: case Op_AddD: case Op_AddL:
case Op_MaxI: case Op_MaxL: case Op_MaxF: case Op_MaxD:
case Op_MinI: case Op_MinL: case Op_MinF: case Op_MinD:
case Op_MaxI: case Op_MaxL: case Op_MaxF: case Op_MaxD: case Op_MaxHF:
case Op_MinI: case Op_MinL: case Op_MinF: case Op_MinD: case Op_MinHF:
case Op_MulI: case Op_MulF: case Op_MulD: case Op_MulL:
case Op_AndL: case Op_XorL: case Op_OrL:
case Op_AndI: case Op_XorI: case Op_OrI: {
Expand Down Expand Up @@ -3787,7 +3787,9 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f
case Op_MulReductionVF:
case Op_MulReductionVD:
case Op_MinReductionV:
case Op_MinReductionVHF:
case Op_MaxReductionV:
case Op_MaxReductionVHF:
case Op_AndReductionV:
case Op_OrReductionV:
case Op_XorReductionV:
Expand Down
Loading