Arm64: Implement LoadAndInsertScalar APIs

src/coreclr/jit/compiler.h

@@ -2814,6 +2814,7 @@ class Compiler
 
 #ifdef TARGET_ARM64
  GenTreeFieldList* gtConvertTableOpToFieldList(GenTree* op, unsigned fieldCount);
+ GenTreeFieldList* gtConvertParamOpToFieldList(GenTree* op, unsigned fieldCount, CORINFO_CLASS_HANDLE clsHnd);
 #endif
 #endif // FEATURE_HW_INTRINSICS
 

src/coreclr/jit/gentree.cpp

@@ -25046,6 +25046,43 @@ GenTreeFieldList* Compiler::gtConvertTableOpToFieldList(GenTree* op, unsigned fi
  }
  return fieldList;
 }
+
+//------------------------------------------------------------------------
+// gtConvertParamOpToFieldList: Convert a operand that represents tuple of struct into
+// field list, where each field represents a struct in the tuple.
+//
+// Arguments:
+// op -- Operand to convert.
+// fieldCount -- Number of fields or rows present.
+// clsHnd -- Class handle of the tuple.
+//
+// Return Value:
+// The GenTreeFieldList node.
+//
+GenTreeFieldList* Compiler::gtConvertParamOpToFieldList(GenTree* op, unsigned fieldCount, CORINFO_CLASS_HANDLE clsHnd)
+{
+ LclVarDsc* opVarDsc = lvaGetDesc(op->AsLclVar());
+ unsigned lclNum = lvaGetLclNum(opVarDsc);
+ unsigned fieldSize = opVarDsc->lvSize() / fieldCount;
+ GenTreeFieldList* fieldList = new (this, GT_FIELD_LIST) GenTreeFieldList();
+ int offset = 0;
+ unsigned sizeBytes = 0;
+ CORINFO_CLASS_HANDLE structType;
+
+ for (unsigned fieldId = 0; fieldId < fieldCount; fieldId++)
+ {
+ CORINFO_FIELD_HANDLE fieldHandle = info.compCompHnd->getFieldInClass(clsHnd, fieldId);
+ JitType2PreciseVarType(info.compCompHnd->getFieldType(fieldHandle, &structType));
+ getBaseJitTypeAndSizeOfSIMDType(structType, &sizeBytes);
+ var_types simdType = getSIMDTypeForSize(sizeBytes);
+
+ GenTreeLclFld* fldNode = gtNewLclFldNode(lclNum, simdType, offset);
+ fieldList->AddField(this, fldNode, offset, simdType);
+
+ offset += fieldSize;
+ }
+ return fieldList;
+}
 #endif // TARGET_ARM64
 
 GenTree* Compiler::gtNewSimdWithLowerNode(
@@ -25195,6 +25232,13 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
 
 #ifdef TARGET_ARM64
  case NI_AdvSimd_LoadAndInsertScalar:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x2:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x3:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x4:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
+
  addr = Op(3);
  break;
 #endif // TARGET_ARM64
@@ -25555,6 +25599,7 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
  case NI_AdvSimd_Arm64_LoadPairVector64:
  case NI_AdvSimd_Arm64_LoadPairVector64NonTemporal:
  case NI_AdvSimd_LoadVector64x2:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x2:
  case NI_AdvSimd_LoadAndReplicateToVector64x2:
  return compiler->typGetBlkLayout(16);
 
@@ -25564,17 +25609,22 @@ ClassLayout* GenTreeHWIntrinsic::GetLayout(Compiler* compiler) const
  case NI_AdvSimd_LoadVector64x4:
  case NI_AdvSimd_LoadAndReplicateToVector64x4:
  case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x4:
  return compiler->typGetBlkLayout(32);
 
  case NI_AdvSimd_LoadVector64x3:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x3:
  case NI_AdvSimd_LoadAndReplicateToVector64x3:
  return compiler->typGetBlkLayout(24);
 
  case NI_AdvSimd_Arm64_LoadVector128x3:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
  case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
  return compiler->typGetBlkLayout(48);
 
  case NI_AdvSimd_Arm64_LoadVector128x4:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
  case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
  return compiler->typGetBlkLayout(64);
 

src/coreclr/jit/hwintrinsic.cpp

@@ -1071,9 +1071,53 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 
  if (HWIntrinsicInfo::IsMultiReg(intrinsic))
  {
- // We don't have generic multireg APIs
  assert(sizeBytes == 0);
  }
+
+#ifdef TARGET_ARM64
+ else if ((intrinsic == NI_AdvSimd_LoadAndInsertScalar) || (intrinsic == NI_AdvSimd_Arm64_LoadAndInsertScalar))
+ {
+ CorInfoType pSimdBaseJitType = CORINFO_TYPE_UNDEF;
+ var_types retFieldType = impNormStructType(sig->retTypeSigClass, &pSimdBaseJitType);
+
+ if (retFieldType == TYP_STRUCT)
+ {
+ CORINFO_CLASS_HANDLE structType;
+ unsigned int sizeBytes = 0;
+
+ // LoadAndInsertScalar that returns 2,3 or 4 vectors
+ assert(pSimdBaseJitType == CORINFO_TYPE_UNDEF);
+ unsigned fieldCount = info.compCompHnd->getClassNumInstanceFields(sig->retTypeSigClass);
+ assert(fieldCount > 1);
+ CORINFO_FIELD_HANDLE fieldHandle = info.compCompHnd->getFieldInClass(sig->retTypeClass, 0);
+ CorInfoType fieldType = info.compCompHnd->getFieldType(fieldHandle, &structType);
+ simdBaseJitType = getBaseJitTypeAndSizeOfSIMDType(structType, &sizeBytes);
+ switch (fieldCount)
+ {
+ case 2:
+ intrinsic = sizeBytes == 8 ? NI_AdvSimd_LoadAndInsertScalarVector64x2
+ : NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2;
+ break;
+ case 3:
+ intrinsic = sizeBytes == 8 ? NI_AdvSimd_LoadAndInsertScalarVector64x3
+ : NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3;
+ break;
+ case 4:
+ intrinsic = sizeBytes == 8 ? NI_AdvSimd_LoadAndInsertScalarVector64x4
+ : NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4;
+ break;
+ default:
+ assert("unsupported");
+ }
+ }
+ else
+ {
+ assert((retFieldType == TYP_SIMD8) || (retFieldType == TYP_SIMD16));
+ assert(isSupportedBaseType(intrinsic, simdBaseJitType));
+ retType = getSIMDTypeForSize(sizeBytes);
+ }
+ }
+#endif
  else
  {
  // We want to return early here for cases where retType was TYP_STRUCT as per method signature and
@@ -1130,7 +1174,9 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
 
 #ifdef TARGET_ARM64
  if ((intrinsic == NI_AdvSimd_Insert) || (intrinsic == NI_AdvSimd_InsertScalar) ||
- (intrinsic == NI_AdvSimd_LoadAndInsertScalar))
+ ((intrinsic >= NI_AdvSimd_LoadAndInsertScalar) && (intrinsic <= NI_AdvSimd_LoadAndInsertScalarVector64x4)) ||
+ ((intrinsic >= NI_AdvSimd_Arm64_LoadAndInsertScalar) &&
+ (intrinsic <= NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4)))
  {
  assert(sig->numArgs == 3);
  immOp = impStackTop(1).val;

src/coreclr/jit/hwintrinsic.h

@@ -770,18 +770,24 @@ struct HWIntrinsicInfo
  case NI_AdvSimd_Arm64_LoadPairVector128NonTemporal:
  case NI_AdvSimd_LoadVector64x2:
  case NI_AdvSimd_Arm64_LoadVector128x2:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x2:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
  case NI_AdvSimd_LoadAndReplicateToVector64x2:
  case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x2:
  return 2;
 
  case NI_AdvSimd_LoadVector64x3:
  case NI_AdvSimd_Arm64_LoadVector128x3:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x3:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
  case NI_AdvSimd_LoadAndReplicateToVector64x3:
  case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x3:
  return 3;
 
  case NI_AdvSimd_LoadVector64x4:
  case NI_AdvSimd_Arm64_LoadVector128x4:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x4:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
  case NI_AdvSimd_LoadAndReplicateToVector64x4:
  case NI_AdvSimd_Arm64_LoadAndReplicateToVector128x4:
  return 4;

src/coreclr/jit/hwintrinsicarm64.cpp

@@ -253,6 +253,12 @@ void HWIntrinsicInfo::lookupImmBounds(
  case NI_AdvSimd_Insert:
  case NI_AdvSimd_InsertScalar:
  case NI_AdvSimd_LoadAndInsertScalar:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x2:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x3:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x4:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
  case NI_AdvSimd_StoreSelectedScalar:
  case NI_AdvSimd_Arm64_DuplicateSelectedScalarToVector128:
  case NI_AdvSimd_Arm64_InsertSelectedScalar:
@@ -1916,6 +1922,57 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
  retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
  break;
  }
+ case NI_AdvSimd_LoadAndInsertScalarVector64x2:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x3:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x4:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
+ {
+ assert(sig->numArgs == 3);
+
+ CORINFO_ARG_LIST_HANDLE arg1 = sig->args;
+ CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(arg1);
+ CORINFO_ARG_LIST_HANDLE arg3 = info.compCompHnd->getArgNext(arg2);
+ var_types argType = TYP_UNKNOWN;
+ CORINFO_CLASS_HANDLE argClass = NO_CLASS_HANDLE;
+
+ argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg3, &argClass)));
+ op3 = getArgForHWIntrinsic(argType, argClass);
+ argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass)));
+ op2 = getArgForHWIntrinsic(argType, argClass);
+ argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg1, &argClass)));
+ op1 = impPopStack().val;
+
+ if (op3->OperIs(GT_CAST))
+ {
+ // Although the API specifies a pointer, if what we have is a BYREF, that's what
+ // we really want, so throw away the cast.
+ if (op3->gtGetOp1()->TypeGet() == TYP_BYREF)
+ {
+ op3 = op3->gtGetOp1();
+ }
+ }
+
+ assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
+ assert(op1->TypeGet() == TYP_STRUCT);
+
+ info.compNeedsConsecutiveRegisters = true;
+ unsigned fieldCount = info.compCompHnd->getClassNumInstanceFields(argClass);
+
+ if (!op1->OperIs(GT_LCL_VAR))
+ {
+ unsigned tmp = lvaGrabTemp(true DEBUGARG("LoadAndInsertScalar temp tree"));
+
+ impStoreTemp(tmp, op1, CHECK_SPILL_NONE);
+ op1 = gtNewLclvNode(tmp, argType);
+ }
+
+ op1 = gtConvertParamOpToFieldList(op1, fieldCount, argClass);
+ op1 = gtNewSimdHWIntrinsicNode(retType, op1, op2, op3, intrinsic, simdBaseJitType, simdSize);
+ retNode = impStoreMultiRegValueToVar(op1, sig->retTypeSigClass DEBUGARG(CorInfoCallConvExtension::Managed));
+ break;
+ }
  case NI_AdvSimd_VectorTableLookup:
  case NI_AdvSimd_Arm64_VectorTableLookup:
  {

src/coreclr/jit/hwintrinsiccodegenarm64.cpp

@@ -728,6 +728,52 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
  }
  break;
 
+ case NI_AdvSimd_LoadAndInsertScalarVector64x2:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x3:
+ case NI_AdvSimd_LoadAndInsertScalarVector64x4:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3:
+ case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4:
+ {
+ assert(isRMW);
+ unsigned fieldIdx = 0;
+ op2Reg = intrin.op2->GetRegNum();
+ op3Reg = intrin.op3->GetRegNum();
+ assert(intrin.op1->OperIsFieldList());
+
+ GenTreeFieldList* fieldList = intrin.op1->AsFieldList();
+ GenTree* firstField = fieldList->Uses().GetHead()->GetNode();
+ op1Reg = firstField->GetRegNum();
+
+ regNumber targetFieldReg = REG_NA;
+ regNumber op1FieldReg = REG_NA;
+
+ for (GenTreeFieldList::Use& use : fieldList->Uses())
+ {
+ GenTree* fieldNode = use.GetNode();
+
+ targetFieldReg = node->GetRegByIndex(fieldIdx);
+ op1FieldReg = fieldNode->GetRegNum();
+
+ if (targetFieldReg != op1FieldReg)
+ {
+ GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(fieldNode), targetFieldReg, op1FieldReg,
+ /* canSkip */ true);
+ }
+ fieldIdx++;
+ }
+
+ HWIntrinsicImmOpHelper helper(this, intrin.op2, node);
+
+ for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd())
+ {
+ const int elementIndex = helper.ImmValue();
+
+ GetEmitter()->emitIns_R_R_I(ins, emitSize, targetReg, op3Reg, elementIndex);
+ }
+
+ break;
+ }
  case NI_AdvSimd_Arm64_LoadPairVector128:
  case NI_AdvSimd_Arm64_LoadPairVector128NonTemporal:
  case NI_AdvSimd_Arm64_LoadPairVector64: