Skip to content

Commit 0af356b

Browse files
merykittySandhya Viswanathan
merykitty
authored and
Sandhya Viswanathan
committed
8278173: [vectorapi] Add x64 intrinsics for unsigned (zero extended) casts
Reviewed-by: psandoz, sviswanathan
1 parent a24498b commit 0af356b

File tree

19 files changed

+466
-62
lines changed

19 files changed

+466
-62
lines changed

‎src/hotspot/cpu/x86/assembler_x86.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4770,11 +4770,20 @@ void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
47704770
assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
47714771
vector_len == AVX_256bit? VM_Version::supports_avx2() :
47724772
vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
4773-
InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4773+
InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
47744774
int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
47754775
emit_int16(0x33, (0xC0 | encode));
47764776
}
47774777

4778+
void Assembler::vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len) {
4779+
assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4780+
vector_len == AVX_256bit? VM_Version::supports_avx2() :
4781+
vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
4782+
InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4783+
int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4784+
emit_int16(0x34, (0xC0 | encode));
4785+
}
4786+
47784787
void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
47794788
NOT_LP64(assert(VM_Version::supports_sse2(), ""));
47804789
InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);

‎src/hotspot/cpu/x86/assembler_x86.hpp‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1822,12 +1822,14 @@ class Assembler : public AbstractAssembler {
18221822
void pmovzxbw(XMMRegister dst, XMMRegister src);
18231823
void pmovzxbw(XMMRegister dst, Address src);
18241824
void pmovzxbd(XMMRegister dst, XMMRegister src);
1825-
void vpmovzxbw( XMMRegister dst, Address src, int vector_len);
1826-
void pmovzxdq(XMMRegister dst, XMMRegister src);
1825+
void vpmovzxbw(XMMRegister dst, Address src, int vector_len);
18271826
void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len);
1828-
void vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len);
18291827
void vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len);
18301828
void vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len);
1829+
void vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len);
1830+
void vpmovzxwq(XMMRegister dst, XMMRegister src, int vector_len);
1831+
void pmovzxdq(XMMRegister dst, XMMRegister src);
1832+
void vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len);
18311833
void evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len);
18321834

18331835
// Sign extend moves
@@ -1844,9 +1846,6 @@ class Assembler : public AbstractAssembler {
18441846

18451847
void evpmovwb(Address dst, XMMRegister src, int vector_len);
18461848
void evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len);
1847-
1848-
void vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len);
1849-
18501849
void evpmovdb(Address dst, XMMRegister src, int vector_len);
18511850

18521851
// Multiply add

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4094,6 +4094,33 @@ void C2_MacroAssembler::vector_castF2I_evex(XMMRegister dst, XMMRegister src, XM
40944094
bind(done);
40954095
}
40964096

4097+
void C2_MacroAssembler::vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
4098+
BasicType from_elem_bt, BasicType to_elem_bt) {
4099+
switch (from_elem_bt) {
4100+
case T_BYTE:
4101+
switch (to_elem_bt) {
4102+
case T_SHORT: vpmovzxbw(dst, src, vlen_enc); break;
4103+
case T_INT: vpmovzxbd(dst, src, vlen_enc); break;
4104+
case T_LONG: vpmovzxbq(dst, src, vlen_enc); break;
4105+
default: ShouldNotReachHere();
4106+
}
4107+
break;
4108+
case T_SHORT:
4109+
switch (to_elem_bt) {
4110+
case T_INT: vpmovzxwd(dst, src, vlen_enc); break;
4111+
case T_LONG: vpmovzxwq(dst, src, vlen_enc); break;
4112+
default: ShouldNotReachHere();
4113+
}
4114+
break;
4115+
case T_INT:
4116+
assert(to_elem_bt == T_LONG, "");
4117+
vpmovzxdq(dst, src, vlen_enc);
4118+
break;
4119+
default:
4120+
ShouldNotReachHere();
4121+
}
4122+
}
4123+
40974124
void C2_MacroAssembler::evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, XMMRegister src3,
40984125
bool merge, BasicType bt, int vlen_enc) {
40994126
if (bt == T_INT) {

‎src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,9 @@
308308
KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
309309
Register scratch, int vec_enc);
310310

311+
void vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
312+
BasicType from_elem_bt, BasicType to_elem_bt);
313+
311314
void evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, XMMRegister src3,
312315
bool merge, BasicType bt, int vlen_enc);
313316

‎src/hotspot/cpu/x86/x86.ad‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1457,6 +1457,9 @@ const bool Matcher::match_rule_supported(int opcode) {
14571457
case Op_VectorCastL2X:
14581458
case Op_VectorCastF2X:
14591459
case Op_VectorCastD2X:
1460+
case Op_VectorUCastB2X:
1461+
case Op_VectorUCastS2X:
1462+
case Op_VectorUCastI2X:
14601463
if (UseAVX < 1) { // enabled for AVX only
14611464
return false;
14621465
}
@@ -7211,6 +7214,22 @@ instruct vcastDtoL_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1,
72117214
ins_pipe( pipe_slow );
72127215
%}
72137216

7217+
instruct vucast(vec dst, vec src) %{
7218+
match(Set dst (VectorUCastB2X src));
7219+
match(Set dst (VectorUCastS2X src));
7220+
match(Set dst (VectorUCastI2X src));
7221+
format %{ "vector_ucast $dst,$src\t!" %}
7222+
ins_encode %{
7223+
assert(UseAVX > 0, "required");
7224+
7225+
BasicType from_elem_bt = Matcher::vector_element_basic_type(this, $src);
7226+
BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
7227+
int vlen_enc = vector_length_encoding(this);
7228+
__ vector_unsigned_cast($dst$$XMMRegister, $src$$XMMRegister, vlen_enc, from_elem_bt, to_elem_bt);
7229+
%}
7230+
ins_pipe( pipe_slow );
7231+
%}
7232+
72147233
// --------------------------------- VectorMaskCmp --------------------------------------
72157234

72167235
instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{

‎src/hotspot/share/adlc/formssel.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4234,6 +4234,7 @@ bool MatchRule::is_vector() const {
42344234
"VectorRearrange","VectorLoadShuffle", "VectorLoadConst",
42354235
"VectorCastB2X", "VectorCastS2X", "VectorCastI2X",
42364236
"VectorCastL2X", "VectorCastF2X", "VectorCastD2X",
4237+
"VectorUCastB2X", "VectorUCastS2X", "VectorUCastI2X",
42374238
"VectorMaskWrapper","VectorMaskCmp","VectorReinterpret","LoadVectorMasked","StoreVectorMasked",
42384239
"FmaVD","FmaVF","PopCountVI", "PopCountVL", "VectorLongToMask",
42394240
// Next are vector mask ops.

‎src/hotspot/share/opto/classes.hpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -479,6 +479,9 @@ macro(VectorCastI2X)
479479
macro(VectorCastL2X)
480480
macro(VectorCastF2X)
481481
macro(VectorCastD2X)
482+
macro(VectorUCastB2X)
483+
macro(VectorUCastS2X)
484+
macro(VectorUCastI2X)
482485
macro(VectorInsert)
483486
macro(MaskAll)
484487
macro(AndVMask)

‎src/hotspot/share/opto/vectorIntrinsics.cpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2370,9 +2370,11 @@ bool LibraryCallKit::inline_vector_convert() {
23702370
return false;
23712371
}
23722372

2373-
assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2373+
assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
2374+
opr->get_con() == VectorSupport::VECTOR_OP_UCAST ||
23742375
opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
2375-
bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
2376+
bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST || opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
2377+
bool is_ucast = (opr->get_con() == VectorSupport::VECTOR_OP_UCAST);
23762378

23772379
ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
23782380
ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
@@ -2457,7 +2459,7 @@ bool LibraryCallKit::inline_vector_convert() {
24572459
if (is_mask && is_floating_point_type(elem_bt_from)) {
24582460
new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG;
24592461
}
2460-
int cast_vopc = VectorCastNode::opcode(new_elem_bt_from);
2462+
int cast_vopc = VectorCastNode::opcode(new_elem_bt_from, !is_ucast);
24612463
// Make sure that cast is implemented to particular type/size combination.
24622464
if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
24632465
if (C->print_intrinsics()) {

‎src/hotspot/share/opto/vectornode.cpp‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1093,23 +1093,27 @@ VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicTyp
10931093
VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
10941094
const TypeVect* vt = TypeVect::make(bt, vlen);
10951095
switch (vopc) {
1096-
case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
1097-
case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
1098-
case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
1099-
case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
1100-
case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
1101-
case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
1096+
case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
1097+
case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
1098+
case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
1099+
case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
1100+
case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
1101+
case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
1102+
case Op_VectorUCastB2X: return new VectorUCastB2XNode(n1, vt);
1103+
case Op_VectorUCastS2X: return new VectorUCastS2XNode(n1, vt);
1104+
case Op_VectorUCastI2X: return new VectorUCastI2XNode(n1, vt);
11021105
default:
11031106
assert(false, "unknown node: %s", NodeClassNames[vopc]);
11041107
return NULL;
11051108
}
11061109
}
11071110

1108-
int VectorCastNode::opcode(BasicType bt) {
1111+
int VectorCastNode::opcode(BasicType bt, bool is_signed) {
1112+
assert((is_integral_type(bt) && bt != T_LONG) || is_signed, "");
11091113
switch (bt) {
1110-
case T_BYTE: return Op_VectorCastB2X;
1111-
case T_SHORT: return Op_VectorCastS2X;
1112-
case T_INT: return Op_VectorCastI2X;
1114+
case T_BYTE: return is_signed ? Op_VectorCastB2X : Op_VectorUCastB2X;
1115+
case T_SHORT: return is_signed ? Op_VectorCastS2X : Op_VectorUCastS2X;
1116+
case T_INT: return is_signed ? Op_VectorCastI2X : Op_VectorUCastI2X;
11131117
case T_LONG: return Op_VectorCastL2X;
11141118
case T_FLOAT: return Op_VectorCastF2X;
11151119
case T_DOUBLE: return Op_VectorCastD2X;

‎src/hotspot/share/opto/vectornode.hpp‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1468,7 +1468,7 @@ class VectorCastNode : public VectorNode {
14681468
virtual int Opcode() const;
14691469

14701470
static VectorCastNode* make(int vopc, Node* n1, BasicType bt, uint vlen);
1471-
static int opcode(BasicType bt);
1471+
static int opcode(BasicType bt, bool is_signed = true);
14721472
static bool implemented(BasicType bt, uint vlen);
14731473

14741474
virtual Node* Identity(PhaseGVN* phase);
@@ -1522,6 +1522,30 @@ class VectorCastD2XNode : public VectorCastNode {
15221522
virtual int Opcode() const;
15231523
};
15241524

1525+
class VectorUCastB2XNode : public VectorCastNode {
1526+
public:
1527+
VectorUCastB2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1528+
assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be byte");
1529+
}
1530+
virtual int Opcode() const;
1531+
};
1532+
1533+
class VectorUCastS2XNode : public VectorCastNode {
1534+
public:
1535+
VectorUCastS2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1536+
assert(in->bottom_type()->is_vect()->element_basic_type() == T_SHORT, "must be short");
1537+
}
1538+
virtual int Opcode() const;
1539+
};
1540+
1541+
class VectorUCastI2XNode : public VectorCastNode {
1542+
public:
1543+
VectorUCastI2XNode(Node* in, const TypeVect* vt) : VectorCastNode(in, vt) {
1544+
assert(in->bottom_type()->is_vect()->element_basic_type() == T_INT, "must be int");
1545+
}
1546+
virtual int Opcode() const;
1547+
};
1548+
15251549
class VectorInsertNode : public VectorNode {
15261550
public:
15271551
VectorInsertNode(Node* vsrc, Node* new_val, ConINode* pos, const TypeVect* vt) : VectorNode(vsrc, new_val, (Node*)pos, vt) {

0 commit comments

Comments
 (0)