diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index d7b6941fcf81d5..42e1c1ce764c4c 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -12074,11 +12074,9 @@ calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth, return std::nullopt; auto VecIdx = IdxOp->getZExtValue(); auto ScalarSize = Op.getScalarValueSizeInBits(); - if (ScalarSize != 32) { + if (ScalarSize < 32) Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index; - } - - return calculateSrcByte(ScalarSize == 32 ? Op : Op.getOperand(0), + return calculateSrcByte(ScalarSize >= 32 ? Op : Op.getOperand(0), StartingIndex, Index); } diff --git a/llvm/test/CodeGen/AMDGPU/match-perm-extract-vector-elt-bug.ll b/llvm/test/CodeGen/AMDGPU/match-perm-extract-vector-elt-bug.ll new file mode 100644 index 00000000000000..c7a831185b83c6 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/match-perm-extract-vector-elt-bug.ll @@ -0,0 +1,109 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 %s -o - | FileCheck -check-prefix=GFX9 %s +; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 %s -o - | FileCheck -check-prefix=GFX10 %s +; RUN: llc -verify-machineinstrs -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 %s -o - | FileCheck -check-prefix=GFX11 %s + +define amdgpu_kernel void @test(ptr addrspace(1) %src, ptr addrspace(1) %dst) { +; GFX9-LABEL: test: +; GFX9: ; %bb.0: ; %entry +; GFX9-NEXT: s_load_dword s7, s[4:5], 0x1c +; GFX9-NEXT: s_load_dword s8, s[4:5], 0x38 +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_and_b32 s4, s7, 0xffff +; GFX9-NEXT: s_mul_i32 s6, s6, s4 +; GFX9-NEXT: s_add_i32 s8, s8, s6 +; GFX9-NEXT: v_add_u32_e32 v0, s8, v0 +; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GFX9-NEXT: v_lshlrev_b64 v[4:5], 4, v[0:1] +; GFX9-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc +; GFX9-NEXT: global_load_dwordx4 v[0:3], v[0:1], off +; GFX9-NEXT: v_mov_b32_e32 v6, s3 +; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, s2, v4 +; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v6, v5, vcc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_not_b32_e32 v3, v3 +; GFX9-NEXT: v_not_b32_e32 v2, v2 +; GFX9-NEXT: v_not_b32_e32 v1, v1 +; GFX9-NEXT: v_not_b32_e32 v0, v0 +; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off +; GFX9-NEXT: s_endpgm +; +; GFX10-LABEL: test: +; GFX10: ; %bb.0: ; %entry +; GFX10-NEXT: s_clause 0x2 +; GFX10-NEXT: s_load_dword s7, s[4:5], 0x1c +; GFX10-NEXT: s_load_dword s8, s[4:5], 0x38 +; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_and_b32 s4, s7, 0xffff +; GFX10-NEXT: s_mul_i32 s6, s6, s4 +; GFX10-NEXT: v_add3_u32 v0, s8, s6, v0 +; GFX10-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GFX10-NEXT: v_lshlrev_b64 v[4:5], 4, v[0:1] +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, s0, v4 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v5, vcc_lo +; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, s2, v4 +; GFX10-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s3, v5, vcc_lo +; GFX10-NEXT: global_load_dwordx4 v[0:3], v[0:1], off +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_not_b32_e32 v3, v3 +; GFX10-NEXT: v_not_b32_e32 v2, v2 +; GFX10-NEXT: v_not_b32_e32 v1, v1 +; GFX10-NEXT: v_not_b32_e32 v0, v0 +; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off +; GFX10-NEXT: s_endpgm +; +; GFX11-LABEL: test: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_clause 0x2 +; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x1c +; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x38 +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_and_b32 s4, s4, 0xffff +; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) +; GFX11-NEXT: s_mul_i32 s15, s15, s4 +; GFX11-NEXT: v_add3_u32 v0, s5, s15, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GFX11-NEXT: v_lshlrev_b64 v[4:5], 4, v[0:1] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, s0, v4 +; GFX11-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s1, v5, vcc_lo +; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, s2, v4 +; GFX11-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, s3, v5, vcc_lo +; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: v_not_b32_e32 v3, v3 +; GFX11-NEXT: v_not_b32_e32 v2, v2 +; GFX11-NEXT: v_not_b32_e32 v1, v1 +; GFX11-NEXT: v_not_b32_e32 v0, v0 +; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off +; GFX11-NEXT: s_nop 0 +; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX11-NEXT: s_endpgm +entry: + %implicitarg.ptr = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() + %arg.1.ptr = getelementptr inbounds i8, ptr addrspace(4) %implicitarg.ptr, i64 40 + %arg.1 = load i64, ptr addrspace(4) %arg.1.ptr, align 8 + %workgroup.id.x = tail call i32 @llvm.amdgcn.workgroup.id.x() + %arg.2.ptr = getelementptr inbounds i8, ptr addrspace(4) %implicitarg.ptr, i64 12 + %arg.2 = load i16, ptr addrspace(4) %arg.2.ptr, align 4 + %arg.2.ext = zext i16 %arg.2 to i32 + %mul = mul i32 %workgroup.id.x, %arg.2.ext + %workitem.id.x = tail call i32 @llvm.amdgcn.workitem.id.x() + %add = add i32 %mul, %workitem.id.x + %add.ext = zext i32 %add to i64 + %add.1 = add i64 %arg.1, %add.ext + %sext = shl i64 %add.1, 32 + %idxprom = ashr exact i64 %sext, 32 + %arrayidx = getelementptr inbounds <16 x i8>, ptr addrspace(1) %src, i64 %idxprom + %arrayval = load <16 x i8>, ptr addrspace(1) %arrayidx, align 16 + %not = xor <16 x i8> %arrayval, + %arrayidx2 = getelementptr inbounds <16 x i8>, ptr addrspace(1) %dst, i64 %idxprom + store <16 x i8> %not, ptr addrspace(1) %arrayidx2, align 16 + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll index 8ac332197215f5..7ca9ae359a4992 100644 --- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll +++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll @@ -3816,13 +3816,15 @@ define hidden void @extract_v13i64(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ; GFX10-LABEL: extract_v13i64: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: global_load_dwordx4 v[8:11], v[0:1], off -; GFX10-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16 +; GFX10-NEXT: s_clause 0x2 +; GFX10-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48 +; GFX10-NEXT: global_load_dwordx4 v[11:14], v[0:1], off +; GFX10-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:64 +; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_perm_b32 v0, v9, v8, 0x3020504 +; GFX10-NEXT: v_perm_b32 v0, v12, v13, 0x1000504 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_perm_b32 v1, v11, v12, 0x1000706 +; GFX10-NEXT: v_perm_b32 v1, v10, v14, 0x1000504 ; GFX10-NEXT: global_store_dword v[4:5], v0, off ; GFX10-NEXT: global_store_dword v[6:7], v1, off ; GFX10-NEXT: s_setpc_b64 s[30:31] @@ -3830,14 +3832,15 @@ define hidden void @extract_v13i64(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ; GFX9-LABEL: extract_v13i64: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[8:11], v[0:1], off -; GFX9-NEXT: global_load_dwordx4 v[12:15], v[0:1], off offset:16 -; GFX9-NEXT: s_mov_b32 s4, 0x3020504 -; GFX9-NEXT: s_mov_b32 s5, 0x1000706 +; GFX9-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:48 +; GFX9-NEXT: global_load_dwordx4 v[11:14], v[0:1], off +; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off offset:64 +; GFX9-NEXT: s_mov_b32 s4, 0x1000504 +; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_perm_b32 v0, v9, v8, s4 +; GFX9-NEXT: v_perm_b32 v0, v12, v13, s4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v1, v11, v12, s5 +; GFX9-NEXT: v_perm_b32 v1, v10, v14, s4 ; GFX9-NEXT: global_store_dword v[4:5], v0, off ; GFX9-NEXT: global_store_dword v[6:7], v1, off ; GFX9-NEXT: s_waitcnt vmcnt(0)