From 3571b0af9c1f4e3493cdfbfb2830b817c996500a Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Mon, 29 Apr 2024 18:27:52 -0400 Subject: [PATCH] [AMDGPU][ISel] Add more trunc store actions regarding bf16 (#90493) Change-Id: Iafcee86aa45b175d1c65fb81d7d788efa3bb0c92 --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 6 + .../CodeGen/AMDGPU/fp_trunc_store_bf16.ll | 426 ------------------ .../AMDGPU/fp_trunc_store_fp32_to_bf16.ll | 145 ++++++ .../AMDGPU/fp_trunc_store_fp64_to_bf16.ll | 142 ++++++ 4 files changed, 293 insertions(+), 426 deletions(-) delete mode 100644 llvm/test/CodeGen/AMDGPU/fp_trunc_store_bf16.ll create mode 100644 llvm/test/CodeGen/AMDGPU/fp_trunc_store_fp32_to_bf16.ll create mode 100644 llvm/test/CodeGen/AMDGPU/fp_trunc_store_fp64_to_bf16.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index d5b1db7f3d4647..7a88de21ab13cf 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -302,6 +302,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, setTruncStoreAction(MVT::f32, MVT::f16, Expand); setTruncStoreAction(MVT::v2f32, MVT::v2bf16, Expand); setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand); + setTruncStoreAction(MVT::v3f32, MVT::v3bf16, Expand); setTruncStoreAction(MVT::v3f32, MVT::v3f16, Expand); setTruncStoreAction(MVT::v4f32, MVT::v4bf16, Expand); setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand); @@ -317,22 +318,27 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, setTruncStoreAction(MVT::f64, MVT::f32, Expand); setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand); + setTruncStoreAction(MVT::v2f64, MVT::v2bf16, Expand); setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand); setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand); setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand); setTruncStoreAction(MVT::v3f64, MVT::v3f32, Expand); + setTruncStoreAction(MVT::v3f64, MVT::v3bf16, Expand); setTruncStoreAction(MVT::v3f64, MVT::v3f16, Expand); setTruncStoreAction(MVT::v4i64, MVT::v4i32, Expand); setTruncStoreAction(MVT::v4i64, MVT::v4i16, Expand); setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand); + setTruncStoreAction(MVT::v4f64, MVT::v4bf16, Expand); setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand); setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand); + setTruncStoreAction(MVT::v8f64, MVT::v8bf16, Expand); setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand); setTruncStoreAction(MVT::v16f64, MVT::v16f32, Expand); + setTruncStoreAction(MVT::v16f64, MVT::v16bf16, Expand); setTruncStoreAction(MVT::v16f64, MVT::v16f16, Expand); setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand); setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand); diff --git a/llvm/test/CodeGen/AMDGPU/fp_trunc_store_bf16.ll b/llvm/test/CodeGen/AMDGPU/fp_trunc_store_bf16.ll deleted file mode 100644 index 5aaff773689f96..00000000000000 --- a/llvm/test/CodeGen/AMDGPU/fp_trunc_store_bf16.ll +++ /dev/null @@ -1,426 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a %s -o - | FileCheck %s - -define void @v2(<2 x float> %num, ptr addrspace(1) %p) { -; CHECK-LABEL: v2: -; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_bfe_u32 v4, v0, 16, 1 -; CHECK-NEXT: s_movk_i32 s4, 0x7fff -; CHECK-NEXT: v_add3_u32 v4, v4, v0, s4 -; CHECK-NEXT: v_or_b32_e32 v5, 0x400000, v0 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc -; CHECK-NEXT: v_bfe_u32 v4, v1, 16, 1 -; CHECK-NEXT: v_add3_u32 v4, v4, v1, s4 -; CHECK-NEXT: v_or_b32_e32 v5, 0x400000, v1 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc -; CHECK-NEXT: s_mov_b32 s4, 0x7060302 -; CHECK-NEXT: v_perm_b32 v0, v1, v0, s4 -; CHECK-NEXT: global_store_dword v[2:3], v0, off -; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: s_setpc_b64 s[30:31] -entry: - %conv = fptrunc <2 x float> %num to <2 x bfloat> - store <2 x bfloat> %conv, ptr addrspace(1) %p, align 8 - ret void -} - -define void @v4(<4 x float> %num, ptr addrspace(1) %p) { -; CHECK-LABEL: v4: -; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_bfe_u32 v6, v2, 16, 1 -; CHECK-NEXT: s_movk_i32 s4, 0x7fff -; CHECK-NEXT: v_add3_u32 v6, v6, v2, s4 -; CHECK-NEXT: v_or_b32_e32 v7, 0x400000, v2 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 -; CHECK-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc -; CHECK-NEXT: v_bfe_u32 v6, v3, 16, 1 -; CHECK-NEXT: v_add3_u32 v6, v6, v3, s4 -; CHECK-NEXT: v_or_b32_e32 v7, 0x400000, v3 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 -; CHECK-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc -; CHECK-NEXT: s_mov_b32 s5, 0x7060302 -; CHECK-NEXT: v_perm_b32 v3, v3, v2, s5 -; CHECK-NEXT: v_bfe_u32 v2, v0, 16, 1 -; CHECK-NEXT: v_add3_u32 v2, v2, v0, s4 -; CHECK-NEXT: v_or_b32_e32 v6, 0x400000, v0 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc -; CHECK-NEXT: v_bfe_u32 v2, v1, 16, 1 -; CHECK-NEXT: v_add3_u32 v2, v2, v1, s4 -; CHECK-NEXT: v_or_b32_e32 v6, 0x400000, v1 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc -; CHECK-NEXT: v_perm_b32 v2, v1, v0, s5 -; CHECK-NEXT: global_store_dwordx2 v[4:5], v[2:3], off -; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: s_setpc_b64 s[30:31] -entry: - %conv = fptrunc <4 x float> %num to <4 x bfloat> - store <4 x bfloat> %conv, ptr addrspace(1) %p, align 8 - ret void -} - -define void @v8(<8 x float> %num, ptr addrspace(1) %p) { -; CHECK-LABEL: v8: -; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_bfe_u32 v10, v6, 16, 1 -; CHECK-NEXT: s_movk_i32 s4, 0x7fff -; CHECK-NEXT: v_add3_u32 v10, v10, v6, s4 -; CHECK-NEXT: v_or_b32_e32 v11, 0x400000, v6 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v6, v6 -; CHECK-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc -; CHECK-NEXT: v_bfe_u32 v10, v7, 16, 1 -; CHECK-NEXT: v_add3_u32 v10, v10, v7, s4 -; CHECK-NEXT: v_or_b32_e32 v11, 0x400000, v7 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 -; CHECK-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc -; CHECK-NEXT: s_mov_b32 s5, 0x7060302 -; CHECK-NEXT: v_perm_b32 v7, v7, v6, s5 -; CHECK-NEXT: v_bfe_u32 v6, v4, 16, 1 -; CHECK-NEXT: v_add3_u32 v6, v6, v4, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v4 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v4, v4 -; CHECK-NEXT: v_cndmask_b32_e32 v4, v6, v10, vcc -; CHECK-NEXT: v_bfe_u32 v6, v5, 16, 1 -; CHECK-NEXT: v_add3_u32 v6, v6, v5, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v5 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v5, v5 -; CHECK-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc -; CHECK-NEXT: v_perm_b32 v6, v5, v4, s5 -; CHECK-NEXT: v_bfe_u32 v4, v2, 16, 1 -; CHECK-NEXT: v_add3_u32 v4, v4, v2, s4 -; CHECK-NEXT: v_or_b32_e32 v5, 0x400000, v2 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 -; CHECK-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc -; CHECK-NEXT: v_bfe_u32 v4, v3, 16, 1 -; CHECK-NEXT: v_add3_u32 v4, v4, v3, s4 -; CHECK-NEXT: v_or_b32_e32 v5, 0x400000, v3 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 -; CHECK-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc -; CHECK-NEXT: v_perm_b32 v5, v3, v2, s5 -; CHECK-NEXT: v_bfe_u32 v2, v0, 16, 1 -; CHECK-NEXT: v_add3_u32 v2, v2, v0, s4 -; CHECK-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc -; CHECK-NEXT: v_bfe_u32 v2, v1, 16, 1 -; CHECK-NEXT: v_add3_u32 v2, v2, v1, s4 -; CHECK-NEXT: v_or_b32_e32 v3, 0x400000, v1 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc -; CHECK-NEXT: v_perm_b32 v4, v1, v0, s5 -; CHECK-NEXT: global_store_dwordx4 v[8:9], v[4:7], off -; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: s_setpc_b64 s[30:31] -entry: - %conv = fptrunc <8 x float> %num to <8 x bfloat> - store <8 x bfloat> %conv, ptr addrspace(1) %p, align 8 - ret void -} - -define void @v16(<16 x float> %num, ptr addrspace(1) %p) { -; CHECK-LABEL: v16: -; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: v_bfe_u32 v18, v6, 16, 1 -; CHECK-NEXT: s_movk_i32 s4, 0x7fff -; CHECK-NEXT: v_add3_u32 v18, v18, v6, s4 -; CHECK-NEXT: v_or_b32_e32 v19, 0x400000, v6 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v6, v6 -; CHECK-NEXT: v_cndmask_b32_e32 v6, v18, v19, vcc -; CHECK-NEXT: v_bfe_u32 v18, v7, 16, 1 -; CHECK-NEXT: v_add3_u32 v18, v18, v7, s4 -; CHECK-NEXT: v_or_b32_e32 v19, 0x400000, v7 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 -; CHECK-NEXT: v_cndmask_b32_e32 v7, v18, v19, vcc -; CHECK-NEXT: s_mov_b32 s5, 0x7060302 -; CHECK-NEXT: v_perm_b32 v7, v7, v6, s5 -; CHECK-NEXT: v_bfe_u32 v6, v4, 16, 1 -; CHECK-NEXT: v_add3_u32 v6, v6, v4, s4 -; CHECK-NEXT: v_or_b32_e32 v18, 0x400000, v4 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v4, v4 -; CHECK-NEXT: v_cndmask_b32_e32 v4, v6, v18, vcc -; CHECK-NEXT: v_bfe_u32 v6, v5, 16, 1 -; CHECK-NEXT: v_add3_u32 v6, v6, v5, s4 -; CHECK-NEXT: v_or_b32_e32 v18, 0x400000, v5 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v5, v5 -; CHECK-NEXT: v_cndmask_b32_e32 v5, v6, v18, vcc -; CHECK-NEXT: v_perm_b32 v6, v5, v4, s5 -; CHECK-NEXT: v_bfe_u32 v4, v2, 16, 1 -; CHECK-NEXT: v_add3_u32 v4, v4, v2, s4 -; CHECK-NEXT: v_or_b32_e32 v5, 0x400000, v2 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 -; CHECK-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc -; CHECK-NEXT: v_bfe_u32 v4, v3, 16, 1 -; CHECK-NEXT: v_add3_u32 v4, v4, v3, s4 -; CHECK-NEXT: v_or_b32_e32 v5, 0x400000, v3 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 -; CHECK-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc -; CHECK-NEXT: v_perm_b32 v5, v3, v2, s5 -; CHECK-NEXT: v_bfe_u32 v2, v0, 16, 1 -; CHECK-NEXT: v_add3_u32 v2, v2, v0, s4 -; CHECK-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc -; CHECK-NEXT: v_bfe_u32 v2, v1, 16, 1 -; CHECK-NEXT: v_add3_u32 v2, v2, v1, s4 -; CHECK-NEXT: v_or_b32_e32 v3, 0x400000, v1 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc -; CHECK-NEXT: v_perm_b32 v4, v1, v0, s5 -; CHECK-NEXT: v_bfe_u32 v0, v14, 16, 1 -; CHECK-NEXT: v_add3_u32 v0, v0, v14, s4 -; CHECK-NEXT: v_or_b32_e32 v1, 0x400000, v14 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v14, v14 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; CHECK-NEXT: v_bfe_u32 v1, v15, 16, 1 -; CHECK-NEXT: v_add3_u32 v1, v1, v15, s4 -; CHECK-NEXT: v_or_b32_e32 v2, 0x400000, v15 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v15, v15 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; CHECK-NEXT: v_perm_b32 v3, v1, v0, s5 -; CHECK-NEXT: v_bfe_u32 v0, v12, 16, 1 -; CHECK-NEXT: v_add3_u32 v0, v0, v12, s4 -; CHECK-NEXT: v_or_b32_e32 v1, 0x400000, v12 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v12, v12 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; CHECK-NEXT: v_bfe_u32 v1, v13, 16, 1 -; CHECK-NEXT: v_add3_u32 v1, v1, v13, s4 -; CHECK-NEXT: v_or_b32_e32 v2, 0x400000, v13 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v13, v13 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; CHECK-NEXT: v_perm_b32 v2, v1, v0, s5 -; CHECK-NEXT: v_bfe_u32 v0, v10, 16, 1 -; CHECK-NEXT: v_add3_u32 v0, v0, v10, s4 -; CHECK-NEXT: v_or_b32_e32 v1, 0x400000, v10 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v10, v10 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; CHECK-NEXT: v_bfe_u32 v1, v11, 16, 1 -; CHECK-NEXT: v_add3_u32 v1, v1, v11, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v11 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v11, v11 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc -; CHECK-NEXT: v_perm_b32 v1, v1, v0, s5 -; CHECK-NEXT: v_bfe_u32 v0, v8, 16, 1 -; CHECK-NEXT: v_add3_u32 v0, v0, v8, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v8 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v8, v8 -; CHECK-NEXT: v_bfe_u32 v8, v9, 16, 1 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc -; CHECK-NEXT: v_add3_u32 v8, v8, v9, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v9 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v9, v9 -; CHECK-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc -; CHECK-NEXT: v_perm_b32 v0, v8, v0, s5 -; CHECK-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:16 -; CHECK-NEXT: global_store_dwordx4 v[16:17], v[4:7], off -; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: s_setpc_b64 s[30:31] -entry: - %conv = fptrunc <16 x float> %num to <16 x bfloat> - store <16 x bfloat> %conv, ptr addrspace(1) %p, align 8 - ret void -} - -define void @v32(<32 x float> %num, ptr addrspace(1) %p) { -; CHECK-LABEL: v32: -; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8 -; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 -; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 -; CHECK-NEXT: v_bfe_u32 v34, v6, 16, 1 -; CHECK-NEXT: s_movk_i32 s4, 0x7fff -; CHECK-NEXT: v_add3_u32 v34, v34, v6, s4 -; CHECK-NEXT: v_or_b32_e32 v35, 0x400000, v6 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v6, v6 -; CHECK-NEXT: v_cndmask_b32_e32 v6, v34, v35, vcc -; CHECK-NEXT: v_bfe_u32 v34, v7, 16, 1 -; CHECK-NEXT: v_add3_u32 v34, v34, v7, s4 -; CHECK-NEXT: v_or_b32_e32 v35, 0x400000, v7 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 -; CHECK-NEXT: v_cndmask_b32_e32 v7, v34, v35, vcc -; CHECK-NEXT: s_mov_b32 s5, 0x7060302 -; CHECK-NEXT: v_perm_b32 v7, v7, v6, s5 -; CHECK-NEXT: v_bfe_u32 v6, v4, 16, 1 -; CHECK-NEXT: v_add3_u32 v6, v6, v4, s4 -; CHECK-NEXT: v_or_b32_e32 v34, 0x400000, v4 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v4, v4 -; CHECK-NEXT: v_cndmask_b32_e32 v4, v6, v34, vcc -; CHECK-NEXT: v_bfe_u32 v6, v5, 16, 1 -; CHECK-NEXT: v_add3_u32 v6, v6, v5, s4 -; CHECK-NEXT: v_or_b32_e32 v34, 0x400000, v5 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v5, v5 -; CHECK-NEXT: v_cndmask_b32_e32 v5, v6, v34, vcc -; CHECK-NEXT: v_perm_b32 v6, v5, v4, s5 -; CHECK-NEXT: v_bfe_u32 v4, v2, 16, 1 -; CHECK-NEXT: v_add3_u32 v4, v4, v2, s4 -; CHECK-NEXT: v_or_b32_e32 v5, 0x400000, v2 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 -; CHECK-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc -; CHECK-NEXT: v_bfe_u32 v4, v3, 16, 1 -; CHECK-NEXT: v_add3_u32 v4, v4, v3, s4 -; CHECK-NEXT: v_or_b32_e32 v5, 0x400000, v3 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 -; CHECK-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc -; CHECK-NEXT: v_perm_b32 v5, v3, v2, s5 -; CHECK-NEXT: v_bfe_u32 v2, v0, 16, 1 -; CHECK-NEXT: v_add3_u32 v2, v2, v0, s4 -; CHECK-NEXT: v_or_b32_e32 v3, 0x400000, v0 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc -; CHECK-NEXT: v_bfe_u32 v2, v1, 16, 1 -; CHECK-NEXT: v_add3_u32 v2, v2, v1, s4 -; CHECK-NEXT: v_or_b32_e32 v3, 0x400000, v1 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc -; CHECK-NEXT: v_perm_b32 v4, v1, v0, s5 -; CHECK-NEXT: v_bfe_u32 v0, v14, 16, 1 -; CHECK-NEXT: v_add3_u32 v0, v0, v14, s4 -; CHECK-NEXT: v_or_b32_e32 v1, 0x400000, v14 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v14, v14 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; CHECK-NEXT: v_bfe_u32 v1, v15, 16, 1 -; CHECK-NEXT: v_add3_u32 v1, v1, v15, s4 -; CHECK-NEXT: v_or_b32_e32 v2, 0x400000, v15 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v15, v15 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; CHECK-NEXT: v_perm_b32 v3, v1, v0, s5 -; CHECK-NEXT: v_bfe_u32 v0, v12, 16, 1 -; CHECK-NEXT: v_add3_u32 v0, v0, v12, s4 -; CHECK-NEXT: v_or_b32_e32 v1, 0x400000, v12 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v12, v12 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; CHECK-NEXT: v_bfe_u32 v1, v13, 16, 1 -; CHECK-NEXT: v_add3_u32 v1, v1, v13, s4 -; CHECK-NEXT: v_or_b32_e32 v2, 0x400000, v13 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v13, v13 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; CHECK-NEXT: v_perm_b32 v2, v1, v0, s5 -; CHECK-NEXT: v_bfe_u32 v0, v10, 16, 1 -; CHECK-NEXT: v_add3_u32 v0, v0, v10, s4 -; CHECK-NEXT: v_or_b32_e32 v1, 0x400000, v10 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v10, v10 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; CHECK-NEXT: v_bfe_u32 v1, v11, 16, 1 -; CHECK-NEXT: v_add3_u32 v1, v1, v11, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v11 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v11, v11 -; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc -; CHECK-NEXT: v_perm_b32 v1, v1, v0, s5 -; CHECK-NEXT: v_bfe_u32 v0, v8, 16, 1 -; CHECK-NEXT: v_add3_u32 v0, v0, v8, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v8 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v8, v8 -; CHECK-NEXT: v_bfe_u32 v8, v9, 16, 1 -; CHECK-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc -; CHECK-NEXT: v_add3_u32 v8, v8, v9, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v9 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v9, v9 -; CHECK-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc -; CHECK-NEXT: v_perm_b32 v0, v8, v0, s5 -; CHECK-NEXT: v_bfe_u32 v8, v22, 16, 1 -; CHECK-NEXT: v_add3_u32 v8, v8, v22, s4 -; CHECK-NEXT: v_or_b32_e32 v9, 0x400000, v22 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v22, v22 -; CHECK-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc -; CHECK-NEXT: v_bfe_u32 v9, v23, 16, 1 -; CHECK-NEXT: v_add3_u32 v9, v9, v23, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v23 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v23, v23 -; CHECK-NEXT: v_cndmask_b32_e32 v9, v9, v10, vcc -; CHECK-NEXT: v_perm_b32 v11, v9, v8, s5 -; CHECK-NEXT: v_bfe_u32 v8, v20, 16, 1 -; CHECK-NEXT: v_add3_u32 v8, v8, v20, s4 -; CHECK-NEXT: v_or_b32_e32 v9, 0x400000, v20 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v20, v20 -; CHECK-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc -; CHECK-NEXT: v_bfe_u32 v9, v21, 16, 1 -; CHECK-NEXT: v_add3_u32 v9, v9, v21, s4 -; CHECK-NEXT: v_or_b32_e32 v10, 0x400000, v21 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v21, v21 -; CHECK-NEXT: v_cndmask_b32_e32 v9, v9, v10, vcc -; CHECK-NEXT: v_perm_b32 v10, v9, v8, s5 -; CHECK-NEXT: v_bfe_u32 v8, v18, 16, 1 -; CHECK-NEXT: v_add3_u32 v8, v8, v18, s4 -; CHECK-NEXT: v_or_b32_e32 v9, 0x400000, v18 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v18, v18 -; CHECK-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc -; CHECK-NEXT: v_bfe_u32 v9, v19, 16, 1 -; CHECK-NEXT: v_add3_u32 v9, v9, v19, s4 -; CHECK-NEXT: v_or_b32_e32 v12, 0x400000, v19 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v19, v19 -; CHECK-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc -; CHECK-NEXT: v_perm_b32 v9, v9, v8, s5 -; CHECK-NEXT: v_bfe_u32 v8, v16, 16, 1 -; CHECK-NEXT: v_add3_u32 v8, v8, v16, s4 -; CHECK-NEXT: v_or_b32_e32 v12, 0x400000, v16 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v16, v16 -; CHECK-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc -; CHECK-NEXT: v_bfe_u32 v12, v17, 16, 1 -; CHECK-NEXT: v_add3_u32 v12, v12, v17, s4 -; CHECK-NEXT: v_or_b32_e32 v13, 0x400000, v17 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v17, v17 -; CHECK-NEXT: v_cndmask_b32_e32 v12, v12, v13, vcc -; CHECK-NEXT: v_perm_b32 v8, v12, v8, s5 -; CHECK-NEXT: v_bfe_u32 v12, v30, 16, 1 -; CHECK-NEXT: v_add3_u32 v12, v12, v30, s4 -; CHECK-NEXT: v_or_b32_e32 v13, 0x400000, v30 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v30, v30 -; CHECK-NEXT: v_cndmask_b32_e32 v12, v12, v13, vcc -; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: v_bfe_u32 v13, v31, 16, 1 -; CHECK-NEXT: v_add3_u32 v13, v13, v31, s4 -; CHECK-NEXT: v_or_b32_e32 v14, 0x400000, v31 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v31, v31 -; CHECK-NEXT: v_cndmask_b32_e32 v13, v13, v14, vcc -; CHECK-NEXT: v_perm_b32 v15, v13, v12, s5 -; CHECK-NEXT: v_bfe_u32 v12, v28, 16, 1 -; CHECK-NEXT: v_add3_u32 v12, v12, v28, s4 -; CHECK-NEXT: v_or_b32_e32 v13, 0x400000, v28 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v28, v28 -; CHECK-NEXT: v_cndmask_b32_e32 v12, v12, v13, vcc -; CHECK-NEXT: v_bfe_u32 v13, v29, 16, 1 -; CHECK-NEXT: v_add3_u32 v13, v13, v29, s4 -; CHECK-NEXT: v_or_b32_e32 v14, 0x400000, v29 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v29, v29 -; CHECK-NEXT: v_cndmask_b32_e32 v13, v13, v14, vcc -; CHECK-NEXT: v_perm_b32 v14, v13, v12, s5 -; CHECK-NEXT: v_bfe_u32 v12, v26, 16, 1 -; CHECK-NEXT: v_add3_u32 v12, v12, v26, s4 -; CHECK-NEXT: v_or_b32_e32 v13, 0x400000, v26 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v26, v26 -; CHECK-NEXT: v_cndmask_b32_e32 v12, v12, v13, vcc -; CHECK-NEXT: v_bfe_u32 v13, v27, 16, 1 -; CHECK-NEXT: v_add3_u32 v13, v13, v27, s4 -; CHECK-NEXT: v_or_b32_e32 v16, 0x400000, v27 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v27, v27 -; CHECK-NEXT: v_cndmask_b32_e32 v13, v13, v16, vcc -; CHECK-NEXT: v_perm_b32 v13, v13, v12, s5 -; CHECK-NEXT: v_bfe_u32 v12, v24, 16, 1 -; CHECK-NEXT: v_add3_u32 v12, v12, v24, s4 -; CHECK-NEXT: v_or_b32_e32 v16, 0x400000, v24 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v24, v24 -; CHECK-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc -; CHECK-NEXT: v_bfe_u32 v16, v25, 16, 1 -; CHECK-NEXT: v_add3_u32 v16, v16, v25, s4 -; CHECK-NEXT: v_or_b32_e32 v17, 0x400000, v25 -; CHECK-NEXT: v_cmp_u_f32_e32 vcc, v25, v25 -; CHECK-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc -; CHECK-NEXT: v_perm_b32 v12, v16, v12, s5 -; CHECK-NEXT: global_store_dwordx4 v[32:33], v[12:15], off offset:48 -; CHECK-NEXT: global_store_dwordx4 v[32:33], v[8:11], off offset:32 -; CHECK-NEXT: global_store_dwordx4 v[32:33], v[0:3], off offset:16 -; CHECK-NEXT: global_store_dwordx4 v[32:33], v[4:7], off -; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: s_setpc_b64 s[30:31] -entry: - %conv = fptrunc <32 x float> %num to <32 x bfloat> - store <32 x bfloat> %conv, ptr addrspace(1) %p, align 8 - ret void -} diff --git a/llvm/test/CodeGen/AMDGPU/fp_trunc_store_fp32_to_bf16.ll b/llvm/test/CodeGen/AMDGPU/fp_trunc_store_fp32_to_bf16.ll new file mode 100644 index 00000000000000..75dc20509adf71 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fp_trunc_store_fp32_to_bf16.ll @@ -0,0 +1,145 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a %s -o - | FileCheck %s + +define void @scalar(float %num, ptr addrspace(1) %p) { +; CHECK-LABEL: scalar: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_mov_b32_e32 v3, v2 +; CHECK-NEXT: v_mov_b32_e32 v2, v1 +; CHECK-NEXT: global_store_short_d16_hi v[2:3], v0, off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc float %num to bfloat + store bfloat %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v2(<2 x float> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v2: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_perm_b32 v0, v1, v0, s4 +; CHECK-NEXT: global_store_dword v[2:3], v0, off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <2 x float> %num to <2 x bfloat> + store <2 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v3(<3 x float> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v3: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_mov_b32_e32 v5, v4 +; CHECK-NEXT: v_mov_b32_e32 v4, v3 +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_perm_b32 v0, v1, v0, s4 +; CHECK-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:4 +; CHECK-NEXT: global_store_dword v[4:5], v0, off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <3 x float> %num to <3 x bfloat> + store <3 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v4(<4 x float> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v4: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_perm_b32 v3, v3, v2, s4 +; CHECK-NEXT: v_perm_b32 v2, v1, v0, s4 +; CHECK-NEXT: global_store_dwordx2 v[4:5], v[2:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <4 x float> %num to <4 x bfloat> + store <4 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v8(<8 x float> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v8: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_perm_b32 v7, v7, v6, s4 +; CHECK-NEXT: v_perm_b32 v6, v5, v4, s4 +; CHECK-NEXT: v_perm_b32 v5, v3, v2, s4 +; CHECK-NEXT: v_perm_b32 v4, v1, v0, s4 +; CHECK-NEXT: global_store_dwordx4 v[8:9], v[4:7], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <8 x float> %num to <8 x bfloat> + store <8 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v16(<16 x float> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v16: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_perm_b32 v7, v7, v6, s4 +; CHECK-NEXT: v_perm_b32 v6, v5, v4, s4 +; CHECK-NEXT: v_perm_b32 v5, v3, v2, s4 +; CHECK-NEXT: v_perm_b32 v4, v1, v0, s4 +; CHECK-NEXT: v_perm_b32 v3, v15, v14, s4 +; CHECK-NEXT: v_perm_b32 v2, v13, v12, s4 +; CHECK-NEXT: v_perm_b32 v1, v11, v10, s4 +; CHECK-NEXT: v_perm_b32 v0, v9, v8, s4 +; CHECK-NEXT: global_store_dwordx4 v[16:17], v[0:3], off offset:16 +; CHECK-NEXT: global_store_dwordx4 v[16:17], v[4:7], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <16 x float> %num to <16 x bfloat> + store <16 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v32(<32 x float> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v32: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 +; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8 +; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_perm_b32 v7, v7, v6, s4 +; CHECK-NEXT: v_perm_b32 v6, v5, v4, s4 +; CHECK-NEXT: v_perm_b32 v4, v1, v0, s4 +; CHECK-NEXT: v_perm_b32 v1, v11, v10, s4 +; CHECK-NEXT: v_perm_b32 v0, v9, v8, s4 +; CHECK-NEXT: v_perm_b32 v11, v23, v22, s4 +; CHECK-NEXT: v_perm_b32 v10, v21, v20, s4 +; CHECK-NEXT: v_perm_b32 v9, v19, v18, s4 +; CHECK-NEXT: v_perm_b32 v8, v17, v16, s4 +; CHECK-NEXT: v_perm_b32 v5, v3, v2, s4 +; CHECK-NEXT: v_perm_b32 v3, v15, v14, s4 +; CHECK-NEXT: v_perm_b32 v2, v13, v12, s4 +; CHECK-NEXT: v_perm_b32 v14, v29, v28, s4 +; CHECK-NEXT: v_perm_b32 v13, v27, v26, s4 +; CHECK-NEXT: v_perm_b32 v12, v25, v24, s4 +; CHECK-NEXT: s_waitcnt vmcnt(2) +; CHECK-NEXT: v_perm_b32 v15, v31, v30, s4 +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: global_store_dwordx4 v[32:33], v[8:11], off offset:32 +; CHECK-NEXT: global_store_dwordx4 v[32:33], v[0:3], off offset:16 +; CHECK-NEXT: global_store_dwordx4 v[32:33], v[12:15], off offset:48 +; CHECK-NEXT: global_store_dwordx4 v[32:33], v[4:7], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <32 x float> %num to <32 x bfloat> + store <32 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/fp_trunc_store_fp64_to_bf16.ll b/llvm/test/CodeGen/AMDGPU/fp_trunc_store_fp64_to_bf16.ll new file mode 100644 index 00000000000000..22b5c86313893d --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/fp_trunc_store_fp64_to_bf16.ll @@ -0,0 +1,142 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a %s -o - | FileCheck %s + +define void @scalar(double %num, ptr addrspace(1) %p) { +; CHECK-LABEL: scalar: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1] +; CHECK-NEXT: global_store_short_d16_hi v[2:3], v0, off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc double %num to bfloat + store bfloat %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v2(<2 x double> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v2: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1] +; CHECK-NEXT: v_cvt_f32_f64_e32 v1, v[2:3] +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_perm_b32 v0, v1, v0, s4 +; CHECK-NEXT: global_store_dword v[4:5], v0, off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <2 x double> %num to <2 x bfloat> + store <2 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v3(<3 x double> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v3: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1] +; CHECK-NEXT: v_cvt_f32_f64_e32 v1, v[2:3] +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_perm_b32 v0, v1, v0, s4 +; CHECK-NEXT: v_cvt_f32_f64_e32 v1, v[4:5] +; CHECK-NEXT: global_store_short_d16_hi v[6:7], v1, off offset:4 +; CHECK-NEXT: global_store_dword v[6:7], v0, off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <3 x double> %num to <3 x bfloat> + store <3 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v4(<4 x double> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v4: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_cvt_f32_f64_e32 v4, v[4:5] +; CHECK-NEXT: v_cvt_f32_f64_e32 v5, v[6:7] +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1] +; CHECK-NEXT: v_cvt_f32_f64_e32 v1, v[2:3] +; CHECK-NEXT: v_perm_b32 v5, v5, v4, s4 +; CHECK-NEXT: v_perm_b32 v4, v1, v0, s4 +; CHECK-NEXT: global_store_dwordx2 v[8:9], v[4:5], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <4 x double> %num to <4 x bfloat> + store <4 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v8(<8 x double> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v8: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: v_cvt_f32_f64_e32 v12, v[12:13] +; CHECK-NEXT: v_cvt_f32_f64_e32 v13, v[14:15] +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_cvt_f32_f64_e32 v8, v[8:9] +; CHECK-NEXT: v_cvt_f32_f64_e32 v9, v[10:11] +; CHECK-NEXT: v_cvt_f32_f64_e32 v4, v[4:5] +; CHECK-NEXT: v_cvt_f32_f64_e32 v5, v[6:7] +; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1] +; CHECK-NEXT: v_cvt_f32_f64_e32 v1, v[2:3] +; CHECK-NEXT: v_perm_b32 v13, v13, v12, s4 +; CHECK-NEXT: v_perm_b32 v12, v9, v8, s4 +; CHECK-NEXT: v_perm_b32 v11, v5, v4, s4 +; CHECK-NEXT: v_perm_b32 v10, v1, v0, s4 +; CHECK-NEXT: global_store_dwordx4 v[16:17], v[10:13], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <8 x double> %num to <8 x bfloat> + store <8 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +} + +define void @v16(<16 x double> %num, ptr addrspace(1) %p) { +; CHECK-LABEL: v16: +; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: buffer_load_dword v31, off, s[0:3], s32 +; CHECK-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8 +; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4 +; CHECK-NEXT: s_mov_b32 s4, 0x7060302 +; CHECK-NEXT: v_cvt_f32_f64_e32 v8, v[8:9] +; CHECK-NEXT: v_cvt_f32_f64_e32 v9, v[10:11] +; CHECK-NEXT: v_cvt_f32_f64_e32 v12, v[12:13] +; CHECK-NEXT: v_cvt_f32_f64_e32 v13, v[14:15] +; CHECK-NEXT: v_cvt_f32_f64_e32 v4, v[4:5] +; CHECK-NEXT: v_cvt_f32_f64_e32 v5, v[6:7] +; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1] +; CHECK-NEXT: v_cvt_f32_f64_e32 v6, v[2:3] +; CHECK-NEXT: v_cvt_f32_f64_e32 v7, v[28:29] +; CHECK-NEXT: v_cvt_f32_f64_e32 v10, v[24:25] +; CHECK-NEXT: v_cvt_f32_f64_e32 v11, v[26:27] +; CHECK-NEXT: v_cvt_f32_f64_e32 v14, v[20:21] +; CHECK-NEXT: v_cvt_f32_f64_e32 v15, v[22:23] +; CHECK-NEXT: v_cvt_f32_f64_e32 v16, v[16:17] +; CHECK-NEXT: v_cvt_f32_f64_e32 v17, v[18:19] +; CHECK-NEXT: v_perm_b32 v2, v9, v8, s4 +; CHECK-NEXT: v_perm_b32 v1, v5, v4, s4 +; CHECK-NEXT: v_perm_b32 v0, v6, v0, s4 +; CHECK-NEXT: v_perm_b32 v6, v11, v10, s4 +; CHECK-NEXT: v_perm_b32 v5, v15, v14, s4 +; CHECK-NEXT: v_perm_b32 v4, v17, v16, s4 +; CHECK-NEXT: v_perm_b32 v3, v13, v12, s4 +; CHECK-NEXT: s_waitcnt vmcnt(2) +; CHECK-NEXT: v_cvt_f32_f64_e32 v8, v[30:31] +; CHECK-NEXT: v_perm_b32 v7, v8, v7, s4 +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: global_store_dwordx4 v[32:33], v[4:7], off offset:16 +; CHECK-NEXT: global_store_dwordx4 v[32:33], v[0:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: s_setpc_b64 s[30:31] +entry: + %conv = fptrunc <16 x double> %num to <16 x bfloat> + store <16 x bfloat> %conv, ptr addrspace(1) %p, align 8 + ret void +}