Google Git
Sign in
fuchsia / third_party / llvm-project / refs/heads/upstream/revert-155826-fix-signed-overflow-dense-map / . / llvm / test / CodeGen / AMDGPU / llvm.amdgcn.mfma.gfx950.ll
blob: 078a043b9460493e04d89ac4885825c613491620 [file] [log] [blame] [edit]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SDAG %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GISEL %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 --amdgpu-mfma-vgpr-form=0 < %s | FileCheck -enable-var-scope --check-prefixes=HEURRC %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx950 -global-isel=0 --amdgpu-mfma-vgpr-form=1 < %s | FileCheck -enable-var-scope --check-prefixes=VGPRRC %s
declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half>, <8 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half>, <8 x half>, <16 x float>, i32 immarg, i32 immarg, i32 immarg)
; --------------------------------------------------------------------
; llvm.amdgcn.mfma.f32.16x16x32.f16
; --------------------------------------------------------------------
define <4 x float> @test_mfma_f32_16x16x32_f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) {
; GCN-LABEL: test_mfma_f32_16x16x32_f16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]
; GCN-NEXT: s_nop 7
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_f32_16x16x32_f16:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_f32_16x16x32_f16:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_f32_16x16x32_f16:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)
ret <4 x float> %result
}
define <4 x float> @test_mfma_f32_16x16x32_f16__flags(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) {
; GCN-LABEL: test_mfma_f32_16x16x32_f16__flags:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; GCN-NEXT: s_nop 7
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_f32_16x16x32_f16__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_f32_16x16x32_f16__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_f32_16x16x32_f16__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 1, i32 1, i32 1)
ret <4 x float> %result
}
define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd(ptr addrspace(1) %out, <8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
; SDAG-NEXT: s_nop 7
; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
; GISEL-NEXT: v_mov_b32_e32 v4, 0
; GISEL-NEXT: s_nop 6
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; AGPR-NEXT: v_mov_b32_e32 v8, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; AGPR-NEXT: v_accvgpr_write_b32 a0, s0
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; AGPR-NEXT: v_accvgpr_write_b32 a1, s1
; AGPR-NEXT: v_accvgpr_write_b32 a2, s2
; AGPR-NEXT: v_accvgpr_write_b32 a3, s3
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3]
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPR-NEXT: v_mov_b32_e32 v12, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPR-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)
store <4 x float> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags(ptr addrspace(1) %out, <8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2) #0 {
; SDAG-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: s_nop 7
; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; GISEL-NEXT: v_mov_b32_e32 v4, 0
; GISEL-NEXT: s_nop 6
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; AGPR-NEXT: v_mov_b32_e32 v8, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; AGPR-NEXT: v_accvgpr_write_b32 a0, s0
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; AGPR-NEXT: v_accvgpr_write_b32 a1, s1
; AGPR-NEXT: v_accvgpr_write_b32 a2, s2
; AGPR-NEXT: v_accvgpr_write_b32 a3, s3
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_16x16x32_f16_no_agpr__vgprcd__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPR-NEXT: v_mov_b32_e32 v12, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_f32_16x16x32_f16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPR-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %arg2, i32 3, i32 2, i32 1)
store <4 x float> %result, ptr addrspace(1) %out
ret void
}
; --------------------------------------------------------------------
; llvm.amdgcn.mfma.f32.32x32x16.f16
; --------------------------------------------------------------------
define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2) #1 {
; SDAG-LABEL: test_mfma_f32_32x32x16_f16:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[8:9], 48
; SDAG-NEXT: v_mov_b64_e32 v[10:11], 32
; SDAG-NEXT: v_mov_b64_e32 v[12:13], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; SDAG-NEXT: v_accvgpr_write_b32 a0, s8
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; SDAG-NEXT: v_accvgpr_write_b32 a1, s9
; SDAG-NEXT: v_accvgpr_write_b32 a2, s10
; SDAG-NEXT: v_accvgpr_write_b32 a3, s11
; SDAG-NEXT: v_accvgpr_write_b32 a4, s12
; SDAG-NEXT: v_accvgpr_write_b32 a5, s13
; SDAG-NEXT: v_accvgpr_write_b32 a6, s14
; SDAG-NEXT: v_accvgpr_write_b32 a7, s15
; SDAG-NEXT: v_accvgpr_write_b32 a8, s16
; SDAG-NEXT: v_accvgpr_write_b32 a9, s17
; SDAG-NEXT: v_accvgpr_write_b32 a10, s18
; SDAG-NEXT: v_accvgpr_write_b32 a11, s19
; SDAG-NEXT: v_accvgpr_write_b32 a12, s20
; SDAG-NEXT: v_accvgpr_write_b32 a13, s21
; SDAG-NEXT: v_accvgpr_write_b32 a14, s22
; SDAG-NEXT: v_accvgpr_write_b32 a15, s23
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
; SDAG-NEXT: v_mov_b32_e32 v2, s22
; SDAG-NEXT: v_mov_b32_e32 v3, s23
; SDAG-NEXT: v_mov_b64_e32 v[14:15], 0
; SDAG-NEXT: s_nop 4
; SDAG-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
; SDAG-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
; SDAG-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_f32_32x32x16_f16:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: v_mov_b64_e32 v[12:13], 0
; GISEL-NEXT: v_mov_b64_e32 v[14:15], 16
; GISEL-NEXT: v_mov_b64_e32 v[16:17], 32
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; GISEL-NEXT: v_accvgpr_write_b32 a0, s8
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; GISEL-NEXT: v_accvgpr_write_b32 a1, s9
; GISEL-NEXT: v_accvgpr_write_b32 a2, s10
; GISEL-NEXT: v_accvgpr_write_b32 a3, s11
; GISEL-NEXT: v_accvgpr_write_b32 a4, s12
; GISEL-NEXT: v_accvgpr_write_b32 a5, s13
; GISEL-NEXT: v_accvgpr_write_b32 a6, s14
; GISEL-NEXT: v_accvgpr_write_b32 a7, s15
; GISEL-NEXT: v_accvgpr_write_b32 a8, s16
; GISEL-NEXT: v_accvgpr_write_b32 a9, s17
; GISEL-NEXT: v_accvgpr_write_b32 a10, s18
; GISEL-NEXT: v_accvgpr_write_b32 a11, s19
; GISEL-NEXT: v_accvgpr_write_b32 a12, s20
; GISEL-NEXT: v_accvgpr_write_b32 a13, s21
; GISEL-NEXT: v_accvgpr_write_b32 a14, s22
; GISEL-NEXT: v_accvgpr_write_b32 a15, s23
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[18:19], 48
; GISEL-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
; GISEL-NEXT: s_nop 7
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
; GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
; GISEL-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_32x32x16_f16:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], 48
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], 32
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 16
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
; HEURRC-NEXT: v_mov_b32_e32 v0, s20
; HEURRC-NEXT: v_mov_b32_e32 v1, s21
; HEURRC-NEXT: v_mov_b32_e32 v2, s22
; HEURRC-NEXT: v_mov_b32_e32 v3, s23
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 0
; HEURRC-NEXT: s_nop 4
; HEURRC-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
; HEURRC-NEXT: v_mov_b32_e32 v2, s10
; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v0, s12
; HEURRC-NEXT: v_mov_b32_e32 v1, s13
; HEURRC-NEXT: v_mov_b32_e32 v2, s14
; HEURRC-NEXT: v_mov_b32_e32 v3, s15
; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], 48
; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], 32
; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: v_mov_b32_e32 v48, s16
; VGPRRC-NEXT: v_mov_b32_e32 v49, s17
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
; VGPRRC-NEXT: v_mov_b32_e32 v50, s18
; VGPRRC-NEXT: v_mov_b32_e32 v51, s19
; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 0
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[48:51], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b64_e32 v[12:13], 48
; AGPR-NEXT: v_mov_b64_e32 v[14:15], 32
; AGPR-NEXT: v_mov_b64_e32 v[16:17], 16
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-NEXT: v_mov_b64_e32 v[18:19], 0
; AGPR-NEXT: v_mov_b32_e32 v8, s16
; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: v_mov_b32_e32 v9, s17
; AGPR-NEXT: v_mov_b32_e32 v10, s18
; AGPR-NEXT: v_mov_b32_e32 v11, s19
; AGPR-NEXT: s_nop 4
; AGPR-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s12
; AGPR-NEXT: v_mov_b32_e32 v1, s13
; AGPR-NEXT: v_mov_b32_e32 v2, s14
; AGPR-NEXT: v_mov_b32_e32 v3, s15
; AGPR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_32x32x16_f16:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b64_e32 v[44:45], 48
; VGPR-NEXT: v_mov_b64_e32 v[46:47], 32
; VGPR-NEXT: v_mov_b64_e32 v[48:49], 16
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[50:51], 0
; VGPR-NEXT: v_mov_b32_e32 v40, s16
; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
; VGPR-NEXT: v_mov_b32_e32 v41, s17
; VGPR-NEXT: v_mov_b32_e32 v42, s18
; VGPR-NEXT: v_mov_b32_e32 v43, s19
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v0, s20
; VGPR-NEXT: v_mov_b32_e32 v1, s21
; VGPR-NEXT: v_mov_b32_e32 v2, s22
; VGPR-NEXT: v_mov_b32_e32 v3, s23
; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s8
; VGPR-NEXT: v_mov_b32_e32 v1, s9
; VGPR-NEXT: v_mov_b32_e32 v2, s10
; VGPR-NEXT: v_mov_b32_e32 v3, s11
; VGPR-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s12
; VGPR-NEXT: v_mov_b32_e32 v1, s13
; VGPR-NEXT: v_mov_b32_e32 v2, s14
; VGPR-NEXT: v_mov_b32_e32 v3, s15
; VGPR-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)
store volatile <16 x float> %result, ptr addrspace(1) null
store volatile <16 x float> %arg2, ptr addrspace(1) null
ret void
}
define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2) #1 {
; SDAG-LABEL: test_mfma_f32_32x32x16_f16__flags:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[8:9], 48
; SDAG-NEXT: v_mov_b64_e32 v[10:11], 32
; SDAG-NEXT: v_mov_b64_e32 v[12:13], 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; SDAG-NEXT: v_accvgpr_write_b32 a0, s8
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; SDAG-NEXT: v_accvgpr_write_b32 a1, s9
; SDAG-NEXT: v_accvgpr_write_b32 a2, s10
; SDAG-NEXT: v_accvgpr_write_b32 a3, s11
; SDAG-NEXT: v_accvgpr_write_b32 a4, s12
; SDAG-NEXT: v_accvgpr_write_b32 a5, s13
; SDAG-NEXT: v_accvgpr_write_b32 a6, s14
; SDAG-NEXT: v_accvgpr_write_b32 a7, s15
; SDAG-NEXT: v_accvgpr_write_b32 a8, s16
; SDAG-NEXT: v_accvgpr_write_b32 a9, s17
; SDAG-NEXT: v_accvgpr_write_b32 a10, s18
; SDAG-NEXT: v_accvgpr_write_b32 a11, s19
; SDAG-NEXT: v_accvgpr_write_b32 a12, s20
; SDAG-NEXT: v_accvgpr_write_b32 a13, s21
; SDAG-NEXT: v_accvgpr_write_b32 a14, s22
; SDAG-NEXT: v_accvgpr_write_b32 a15, s23
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
; SDAG-NEXT: v_mov_b32_e32 v0, s20
; SDAG-NEXT: v_mov_b32_e32 v1, s21
; SDAG-NEXT: v_mov_b32_e32 v2, s22
; SDAG-NEXT: v_mov_b32_e32 v3, s23
; SDAG-NEXT: v_mov_b64_e32 v[14:15], 0
; SDAG-NEXT: s_nop 4
; SDAG-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
; SDAG-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
; SDAG-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_f32_32x32x16_f16__flags:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: v_mov_b64_e32 v[12:13], 0
; GISEL-NEXT: v_mov_b64_e32 v[14:15], 16
; GISEL-NEXT: v_mov_b64_e32 v[16:17], 32
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; GISEL-NEXT: v_accvgpr_write_b32 a0, s8
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; GISEL-NEXT: v_accvgpr_write_b32 a1, s9
; GISEL-NEXT: v_accvgpr_write_b32 a2, s10
; GISEL-NEXT: v_accvgpr_write_b32 a3, s11
; GISEL-NEXT: v_accvgpr_write_b32 a4, s12
; GISEL-NEXT: v_accvgpr_write_b32 a5, s13
; GISEL-NEXT: v_accvgpr_write_b32 a6, s14
; GISEL-NEXT: v_accvgpr_write_b32 a7, s15
; GISEL-NEXT: v_accvgpr_write_b32 a8, s16
; GISEL-NEXT: v_accvgpr_write_b32 a9, s17
; GISEL-NEXT: v_accvgpr_write_b32 a10, s18
; GISEL-NEXT: v_accvgpr_write_b32 a11, s19
; GISEL-NEXT: v_accvgpr_write_b32 a12, s20
; GISEL-NEXT: v_accvgpr_write_b32 a13, s21
; GISEL-NEXT: v_accvgpr_write_b32 a14, s22
; GISEL-NEXT: v_accvgpr_write_b32 a15, s23
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[18:19], 48
; GISEL-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
; GISEL-NEXT: s_nop 7
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
; GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
; GISEL-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], 48
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], 32
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], 16
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; HEURRC-NEXT: v_accvgpr_write_b32 a0, s8
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; HEURRC-NEXT: v_accvgpr_write_b32 a1, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, s10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, s11
; HEURRC-NEXT: v_accvgpr_write_b32 a4, s12
; HEURRC-NEXT: v_accvgpr_write_b32 a5, s13
; HEURRC-NEXT: v_accvgpr_write_b32 a6, s14
; HEURRC-NEXT: v_accvgpr_write_b32 a7, s15
; HEURRC-NEXT: v_accvgpr_write_b32 a8, s16
; HEURRC-NEXT: v_accvgpr_write_b32 a9, s17
; HEURRC-NEXT: v_accvgpr_write_b32 a10, s18
; HEURRC-NEXT: v_accvgpr_write_b32 a11, s19
; HEURRC-NEXT: v_accvgpr_write_b32 a12, s20
; HEURRC-NEXT: v_accvgpr_write_b32 a13, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a14, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, s23
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
; HEURRC-NEXT: v_mov_b32_e32 v0, s20
; HEURRC-NEXT: v_mov_b32_e32 v1, s21
; HEURRC-NEXT: v_mov_b32_e32 v2, s22
; HEURRC-NEXT: v_mov_b32_e32 v3, s23
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], 0
; HEURRC-NEXT: s_nop 4
; HEURRC-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[10:11], v[16:19], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
; HEURRC-NEXT: v_mov_b32_e32 v2, s10
; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v0, s12
; HEURRC-NEXT: v_mov_b32_e32 v1, s13
; HEURRC-NEXT: v_mov_b32_e32 v2, s14
; HEURRC-NEXT: v_mov_b32_e32 v3, s15
; HEURRC-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[40:41], 48
; VGPRRC-NEXT: v_mov_b64_e32 v[42:43], 32
; VGPRRC-NEXT: v_mov_b64_e32 v[44:45], 16
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: v_mov_b32_e32 v48, s16
; VGPRRC-NEXT: v_mov_b32_e32 v49, s17
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
; VGPRRC-NEXT: v_mov_b32_e32 v50, s18
; VGPRRC-NEXT: v_mov_b32_e32 v51, s19
; VGPRRC-NEXT: v_mov_b64_e32 v[46:47], 0
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
; VGPRRC-NEXT: global_store_dwordx4 v[42:43], v[48:51], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
; VGPRRC-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
; VGPRRC-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b64_e32 v[12:13], 48
; AGPR-NEXT: v_mov_b64_e32 v[14:15], 32
; AGPR-NEXT: v_mov_b64_e32 v[16:17], 16
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-NEXT: v_mov_b64_e32 v[18:19], 0
; AGPR-NEXT: v_mov_b32_e32 v8, s16
; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: v_mov_b32_e32 v9, s17
; AGPR-NEXT: v_mov_b32_e32 v10, s18
; AGPR-NEXT: v_mov_b32_e32 v11, s19
; AGPR-NEXT: s_nop 4
; AGPR-NEXT: global_store_dwordx4 v[12:13], a[28:31], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[14:15], a[24:27], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[16:17], a[20:23], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[18:19], a[16:19], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[14:15], v[8:11], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s12
; AGPR-NEXT: v_mov_b32_e32 v1, s13
; AGPR-NEXT: v_mov_b32_e32 v2, s14
; AGPR-NEXT: v_mov_b32_e32 v3, s15
; AGPR-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_32x32x16_f16__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b64_e32 v[44:45], 48
; VGPR-NEXT: v_mov_b64_e32 v[46:47], 32
; VGPR-NEXT: v_mov_b64_e32 v[48:49], 16
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[50:51], 0
; VGPR-NEXT: v_mov_b32_e32 v40, s16
; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
; VGPR-NEXT: v_mov_b32_e32 v41, s17
; VGPR-NEXT: v_mov_b32_e32 v42, s18
; VGPR-NEXT: v_mov_b32_e32 v43, s19
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: global_store_dwordx4 v[44:45], v[28:31], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[46:47], v[24:27], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[48:49], v[20:23], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[50:51], v[16:19], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[46:47], v[40:43], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v0, s20
; VGPR-NEXT: v_mov_b32_e32 v1, s21
; VGPR-NEXT: v_mov_b32_e32 v2, s22
; VGPR-NEXT: v_mov_b32_e32 v3, s23
; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s8
; VGPR-NEXT: v_mov_b32_e32 v1, s9
; VGPR-NEXT: v_mov_b32_e32 v2, s10
; VGPR-NEXT: v_mov_b32_e32 v3, s11
; VGPR-NEXT: global_store_dwordx4 v[50:51], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s12
; VGPR-NEXT: v_mov_b32_e32 v1, s13
; VGPR-NEXT: v_mov_b32_e32 v2, s14
; VGPR-NEXT: v_mov_b32_e32 v3, s15
; VGPR-NEXT: global_store_dwordx4 v[48:49], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 2, i32 3, i32 1)
store volatile <16 x float> %result, ptr addrspace(1) null
store volatile <16 x float> %arg2, ptr addrspace(1) null
ret void
}
define <16 x float> @test_mfma_f32_32x32x16_f16__mac(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2) {
; GCN-LABEL: test_mfma_f32_32x32x16_f16__mac:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: v_accvgpr_write_b32 a4, v12
; GCN-NEXT: v_accvgpr_write_b32 a5, v13
; GCN-NEXT: v_accvgpr_write_b32 a6, v14
; GCN-NEXT: v_accvgpr_write_b32 a7, v15
; GCN-NEXT: v_accvgpr_write_b32 a8, v16
; GCN-NEXT: v_accvgpr_write_b32 a9, v17
; GCN-NEXT: v_accvgpr_write_b32 a10, v18
; GCN-NEXT: v_accvgpr_write_b32 a11, v19
; GCN-NEXT: v_accvgpr_write_b32 a12, v20
; GCN-NEXT: v_accvgpr_write_b32 a13, v21
; GCN-NEXT: v_accvgpr_write_b32 a14, v22
; GCN-NEXT: v_accvgpr_write_b32 a15, v23
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]
; GCN-NEXT: s_nop 7
; GCN-NEXT: s_nop 3
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: v_accvgpr_read_b32 v4, a4
; GCN-NEXT: v_accvgpr_read_b32 v5, a5
; GCN-NEXT: v_accvgpr_read_b32 v6, a6
; GCN-NEXT: v_accvgpr_read_b32 v7, a7
; GCN-NEXT: v_accvgpr_read_b32 v8, a8
; GCN-NEXT: v_accvgpr_read_b32 v9, a9
; GCN-NEXT: v_accvgpr_read_b32 v10, a10
; GCN-NEXT: v_accvgpr_read_b32 v11, a11
; GCN-NEXT: v_accvgpr_read_b32 v12, a12
; GCN-NEXT: v_accvgpr_read_b32 v13, a13
; GCN-NEXT: v_accvgpr_read_b32 v14, a14
; GCN-NEXT: v_accvgpr_read_b32 v15, a15
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__mac:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: v_accvgpr_write_b32 a4, v12
; HEURRC-NEXT: v_accvgpr_write_b32 a5, v13
; HEURRC-NEXT: v_accvgpr_write_b32 a6, v14
; HEURRC-NEXT: v_accvgpr_write_b32 a7, v15
; HEURRC-NEXT: v_accvgpr_write_b32 a8, v16
; HEURRC-NEXT: v_accvgpr_write_b32 a9, v17
; HEURRC-NEXT: v_accvgpr_write_b32 a10, v18
; HEURRC-NEXT: v_accvgpr_write_b32 a11, v19
; HEURRC-NEXT: v_accvgpr_write_b32 a12, v20
; HEURRC-NEXT: v_accvgpr_write_b32 a13, v21
; HEURRC-NEXT: v_accvgpr_write_b32 a14, v22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, v23
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: s_nop 3
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: v_accvgpr_read_b32 v4, a4
; HEURRC-NEXT: v_accvgpr_read_b32 v5, a5
; HEURRC-NEXT: v_accvgpr_read_b32 v6, a6
; HEURRC-NEXT: v_accvgpr_read_b32 v7, a7
; HEURRC-NEXT: v_accvgpr_read_b32 v8, a8
; HEURRC-NEXT: v_accvgpr_read_b32 v9, a9
; HEURRC-NEXT: v_accvgpr_read_b32 v10, a10
; HEURRC-NEXT: v_accvgpr_read_b32 v11, a11
; HEURRC-NEXT: v_accvgpr_read_b32 v12, a12
; HEURRC-NEXT: v_accvgpr_read_b32 v13, a13
; HEURRC-NEXT: v_accvgpr_read_b32 v14, a14
; HEURRC-NEXT: v_accvgpr_read_b32 v15, a15
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__mac:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23]
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 3
; VGPRRC-NEXT: v_mov_b32_e32 v0, v8
; VGPRRC-NEXT: v_mov_b32_e32 v1, v9
; VGPRRC-NEXT: v_mov_b32_e32 v2, v10
; VGPRRC-NEXT: v_mov_b32_e32 v3, v11
; VGPRRC-NEXT: v_mov_b32_e32 v4, v12
; VGPRRC-NEXT: v_mov_b32_e32 v5, v13
; VGPRRC-NEXT: v_mov_b32_e32 v6, v14
; VGPRRC-NEXT: v_mov_b32_e32 v7, v15
; VGPRRC-NEXT: v_mov_b32_e32 v8, v16
; VGPRRC-NEXT: v_mov_b32_e32 v9, v17
; VGPRRC-NEXT: v_mov_b32_e32 v10, v18
; VGPRRC-NEXT: v_mov_b32_e32 v11, v19
; VGPRRC-NEXT: v_mov_b32_e32 v12, v20
; VGPRRC-NEXT: v_mov_b32_e32 v13, v21
; VGPRRC-NEXT: v_mov_b32_e32 v14, v22
; VGPRRC-NEXT: v_mov_b32_e32 v15, v23
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__mac:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: v_accvgpr_write_b32 a4, v12
; AGPR-NEXT: v_accvgpr_write_b32 a5, v13
; AGPR-NEXT: v_accvgpr_write_b32 a6, v14
; AGPR-NEXT: v_accvgpr_write_b32 a7, v15
; AGPR-NEXT: v_accvgpr_write_b32 a8, v16
; AGPR-NEXT: v_accvgpr_write_b32 a9, v17
; AGPR-NEXT: v_accvgpr_write_b32 a10, v18
; AGPR-NEXT: v_accvgpr_write_b32 a11, v19
; AGPR-NEXT: v_accvgpr_write_b32 a12, v20
; AGPR-NEXT: v_accvgpr_write_b32 a13, v21
; AGPR-NEXT: v_accvgpr_write_b32 a14, v22
; AGPR-NEXT: v_accvgpr_write_b32 a15, v23
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: s_nop 3
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: v_accvgpr_read_b32 v4, a4
; AGPR-NEXT: v_accvgpr_read_b32 v5, a5
; AGPR-NEXT: v_accvgpr_read_b32 v6, a6
; AGPR-NEXT: v_accvgpr_read_b32 v7, a7
; AGPR-NEXT: v_accvgpr_read_b32 v8, a8
; AGPR-NEXT: v_accvgpr_read_b32 v9, a9
; AGPR-NEXT: v_accvgpr_read_b32 v10, a10
; AGPR-NEXT: v_accvgpr_read_b32 v11, a11
; AGPR-NEXT: v_accvgpr_read_b32 v12, a12
; AGPR-NEXT: v_accvgpr_read_b32 v13, a13
; AGPR-NEXT: v_accvgpr_read_b32 v14, a14
; AGPR-NEXT: v_accvgpr_read_b32 v15, a15
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_f32_32x32x16_f16__mac:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23]
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 3
; VGPR-NEXT: v_mov_b32_e32 v0, v8
; VGPR-NEXT: v_mov_b32_e32 v1, v9
; VGPR-NEXT: v_mov_b32_e32 v2, v10
; VGPR-NEXT: v_mov_b32_e32 v3, v11
; VGPR-NEXT: v_mov_b32_e32 v4, v12
; VGPR-NEXT: v_mov_b32_e32 v5, v13
; VGPR-NEXT: v_mov_b32_e32 v6, v14
; VGPR-NEXT: v_mov_b32_e32 v7, v15
; VGPR-NEXT: v_mov_b32_e32 v8, v16
; VGPR-NEXT: v_mov_b32_e32 v9, v17
; VGPR-NEXT: v_mov_b32_e32 v10, v18
; VGPR-NEXT: v_mov_b32_e32 v11, v19
; VGPR-NEXT: v_mov_b32_e32 v12, v20
; VGPR-NEXT: v_mov_b32_e32 v13, v21
; VGPR-NEXT: v_mov_b32_e32 v14, v22
; VGPR-NEXT: v_mov_b32_e32 v15, v23
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)
ret <16 x float> %result
}
define <16 x float> @test_mfma_f32_32x32x16_f16__mac__flags(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2) {
; GCN-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: v_accvgpr_write_b32 a4, v12
; GCN-NEXT: v_accvgpr_write_b32 a5, v13
; GCN-NEXT: v_accvgpr_write_b32 a6, v14
; GCN-NEXT: v_accvgpr_write_b32 a7, v15
; GCN-NEXT: v_accvgpr_write_b32 a8, v16
; GCN-NEXT: v_accvgpr_write_b32 a9, v17
; GCN-NEXT: v_accvgpr_write_b32 a10, v18
; GCN-NEXT: v_accvgpr_write_b32 a11, v19
; GCN-NEXT: v_accvgpr_write_b32 a12, v20
; GCN-NEXT: v_accvgpr_write_b32 a13, v21
; GCN-NEXT: v_accvgpr_write_b32 a14, v22
; GCN-NEXT: v_accvgpr_write_b32 a15, v23
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
; GCN-NEXT: s_nop 7
; GCN-NEXT: s_nop 3
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: v_accvgpr_read_b32 v4, a4
; GCN-NEXT: v_accvgpr_read_b32 v5, a5
; GCN-NEXT: v_accvgpr_read_b32 v6, a6
; GCN-NEXT: v_accvgpr_read_b32 v7, a7
; GCN-NEXT: v_accvgpr_read_b32 v8, a8
; GCN-NEXT: v_accvgpr_read_b32 v9, a9
; GCN-NEXT: v_accvgpr_read_b32 v10, a10
; GCN-NEXT: v_accvgpr_read_b32 v11, a11
; GCN-NEXT: v_accvgpr_read_b32 v12, a12
; GCN-NEXT: v_accvgpr_read_b32 v13, a13
; GCN-NEXT: v_accvgpr_read_b32 v14, a14
; GCN-NEXT: v_accvgpr_read_b32 v15, a15
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: v_accvgpr_write_b32 a4, v12
; HEURRC-NEXT: v_accvgpr_write_b32 a5, v13
; HEURRC-NEXT: v_accvgpr_write_b32 a6, v14
; HEURRC-NEXT: v_accvgpr_write_b32 a7, v15
; HEURRC-NEXT: v_accvgpr_write_b32 a8, v16
; HEURRC-NEXT: v_accvgpr_write_b32 a9, v17
; HEURRC-NEXT: v_accvgpr_write_b32 a10, v18
; HEURRC-NEXT: v_accvgpr_write_b32 a11, v19
; HEURRC-NEXT: v_accvgpr_write_b32 a12, v20
; HEURRC-NEXT: v_accvgpr_write_b32 a13, v21
; HEURRC-NEXT: v_accvgpr_write_b32 a14, v22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, v23
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: s_nop 3
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: v_accvgpr_read_b32 v4, a4
; HEURRC-NEXT: v_accvgpr_read_b32 v5, a5
; HEURRC-NEXT: v_accvgpr_read_b32 v6, a6
; HEURRC-NEXT: v_accvgpr_read_b32 v7, a7
; HEURRC-NEXT: v_accvgpr_read_b32 v8, a8
; HEURRC-NEXT: v_accvgpr_read_b32 v9, a9
; HEURRC-NEXT: v_accvgpr_read_b32 v10, a10
; HEURRC-NEXT: v_accvgpr_read_b32 v11, a11
; HEURRC-NEXT: v_accvgpr_read_b32 v12, a12
; HEURRC-NEXT: v_accvgpr_read_b32 v13, a13
; HEURRC-NEXT: v_accvgpr_read_b32 v14, a14
; HEURRC-NEXT: v_accvgpr_read_b32 v15, a15
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:1
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 3
; VGPRRC-NEXT: v_mov_b32_e32 v0, v8
; VGPRRC-NEXT: v_mov_b32_e32 v1, v9
; VGPRRC-NEXT: v_mov_b32_e32 v2, v10
; VGPRRC-NEXT: v_mov_b32_e32 v3, v11
; VGPRRC-NEXT: v_mov_b32_e32 v4, v12
; VGPRRC-NEXT: v_mov_b32_e32 v5, v13
; VGPRRC-NEXT: v_mov_b32_e32 v6, v14
; VGPRRC-NEXT: v_mov_b32_e32 v7, v15
; VGPRRC-NEXT: v_mov_b32_e32 v8, v16
; VGPRRC-NEXT: v_mov_b32_e32 v9, v17
; VGPRRC-NEXT: v_mov_b32_e32 v10, v18
; VGPRRC-NEXT: v_mov_b32_e32 v11, v19
; VGPRRC-NEXT: v_mov_b32_e32 v12, v20
; VGPRRC-NEXT: v_mov_b32_e32 v13, v21
; VGPRRC-NEXT: v_mov_b32_e32 v14, v22
; VGPRRC-NEXT: v_mov_b32_e32 v15, v23
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: v_accvgpr_write_b32 a4, v12
; AGPR-NEXT: v_accvgpr_write_b32 a5, v13
; AGPR-NEXT: v_accvgpr_write_b32 a6, v14
; AGPR-NEXT: v_accvgpr_write_b32 a7, v15
; AGPR-NEXT: v_accvgpr_write_b32 a8, v16
; AGPR-NEXT: v_accvgpr_write_b32 a9, v17
; AGPR-NEXT: v_accvgpr_write_b32 a10, v18
; AGPR-NEXT: v_accvgpr_write_b32 a11, v19
; AGPR-NEXT: v_accvgpr_write_b32 a12, v20
; AGPR-NEXT: v_accvgpr_write_b32 a13, v21
; AGPR-NEXT: v_accvgpr_write_b32 a14, v22
; AGPR-NEXT: v_accvgpr_write_b32 a15, v23
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: s_nop 3
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: v_accvgpr_read_b32 v4, a4
; AGPR-NEXT: v_accvgpr_read_b32 v5, a5
; AGPR-NEXT: v_accvgpr_read_b32 v6, a6
; AGPR-NEXT: v_accvgpr_read_b32 v7, a7
; AGPR-NEXT: v_accvgpr_read_b32 v8, a8
; AGPR-NEXT: v_accvgpr_read_b32 v9, a9
; AGPR-NEXT: v_accvgpr_read_b32 v10, a10
; AGPR-NEXT: v_accvgpr_read_b32 v11, a11
; AGPR-NEXT: v_accvgpr_read_b32 v12, a12
; AGPR-NEXT: v_accvgpr_read_b32 v13, a13
; AGPR-NEXT: v_accvgpr_read_b32 v14, a14
; AGPR-NEXT: v_accvgpr_read_b32 v15, a15
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_f32_32x32x16_f16__mac__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:1
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 3
; VGPR-NEXT: v_mov_b32_e32 v0, v8
; VGPR-NEXT: v_mov_b32_e32 v1, v9
; VGPR-NEXT: v_mov_b32_e32 v2, v10
; VGPR-NEXT: v_mov_b32_e32 v3, v11
; VGPR-NEXT: v_mov_b32_e32 v4, v12
; VGPR-NEXT: v_mov_b32_e32 v5, v13
; VGPR-NEXT: v_mov_b32_e32 v6, v14
; VGPR-NEXT: v_mov_b32_e32 v7, v15
; VGPR-NEXT: v_mov_b32_e32 v8, v16
; VGPR-NEXT: v_mov_b32_e32 v9, v17
; VGPR-NEXT: v_mov_b32_e32 v10, v18
; VGPR-NEXT: v_mov_b32_e32 v11, v19
; VGPR-NEXT: v_mov_b32_e32 v12, v20
; VGPR-NEXT: v_mov_b32_e32 v13, v21
; VGPR-NEXT: v_mov_b32_e32 v14, v22
; VGPR-NEXT: v_mov_b32_e32 v15, v23
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 1, i32 1, i32 1)
ret <16 x float> %result
}
define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v44, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v40, s20
; SDAG-NEXT: v_mov_b32_e32 v41, s21
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]
; SDAG-NEXT: v_mov_b32_e32 v42, s22
; SDAG-NEXT: v_mov_b32_e32 v43, s23
; SDAG-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
; SDAG-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
; SDAG-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
; SDAG-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; GISEL-NEXT: v_mov_b32_e32 v56, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v44, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; HEURRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: v_mov_b32_e32 v40, s20
; HEURRC-NEXT: v_mov_b32_e32 v41, s21
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]
; HEURRC-NEXT: v_mov_b32_e32 v42, s22
; HEURRC-NEXT: v_mov_b32_e32 v43, s23
; HEURRC-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
; HEURRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
; HEURRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
; HEURRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v44, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: v_mov_b32_e32 v40, s20
; VGPRRC-NEXT: v_mov_b32_e32 v41, s21
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]
; VGPRRC-NEXT: v_mov_b32_e32 v42, s22
; VGPRRC-NEXT: v_mov_b32_e32 v43, s23
; VGPRRC-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; AGPR-NEXT: v_mov_b32_e32 v12, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; AGPR-NEXT: v_accvgpr_write_b32 a31, s23
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; AGPR-NEXT: v_accvgpr_write_b32 a30, s22
; AGPR-NEXT: v_accvgpr_write_b32 a29, s21
; AGPR-NEXT: v_accvgpr_write_b32 a28, s20
; AGPR-NEXT: v_accvgpr_write_b32 a27, s19
; AGPR-NEXT: v_accvgpr_write_b32 a26, s18
; AGPR-NEXT: v_accvgpr_write_b32 a25, s17
; AGPR-NEXT: v_accvgpr_write_b32 a24, s16
; AGPR-NEXT: v_accvgpr_write_b32 a23, s15
; AGPR-NEXT: v_accvgpr_write_b32 a22, s14
; AGPR-NEXT: v_accvgpr_write_b32 a21, s13
; AGPR-NEXT: v_accvgpr_write_b32 a20, s12
; AGPR-NEXT: v_accvgpr_write_b32 a19, s11
; AGPR-NEXT: v_accvgpr_write_b32 a18, s10
; AGPR-NEXT: v_accvgpr_write_b32 a17, s9
; AGPR-NEXT: v_accvgpr_write_b32 a16, s8
; AGPR-NEXT: v_mov_b32_e32 v8, s20
; AGPR-NEXT: v_mov_b32_e32 v9, s21
; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31]
; AGPR-NEXT: v_mov_b32_e32 v10, s22
; AGPR-NEXT: v_mov_b32_e32 v11, s23
; AGPR-NEXT: v_mov_b32_e32 v0, s16
; AGPR-NEXT: v_mov_b32_e32 v1, s17
; AGPR-NEXT: v_mov_b32_e32 v2, s18
; AGPR-NEXT: v_mov_b32_e32 v3, s19
; AGPR-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s12
; AGPR-NEXT: v_mov_b32_e32 v1, s13
; AGPR-NEXT: v_mov_b32_e32 v2, s14
; AGPR-NEXT: v_mov_b32_e32 v3, s15
; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPR-NEXT: v_mov_b32_e32 v44, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPR-NEXT: v_mov_b32_e32 v40, s20
; VGPR-NEXT: v_mov_b32_e32 v41, s21
; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31]
; VGPR-NEXT: v_mov_b32_e32 v42, s22
; VGPR-NEXT: v_mov_b32_e32 v43, s23
; VGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 2
; VGPR-NEXT: v_mov_b32_e32 v16, s16
; VGPR-NEXT: v_mov_b32_e32 v17, s17
; VGPR-NEXT: v_mov_b32_e32 v18, s18
; VGPR-NEXT: v_mov_b32_e32 v19, s19
; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s12
; VGPR-NEXT: v_mov_b32_e32 v17, s13
; VGPR-NEXT: v_mov_b32_e32 v18, s14
; VGPR-NEXT: v_mov_b32_e32 v19, s15
; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s8
; VGPR-NEXT: v_mov_b32_e32 v17, s9
; VGPR-NEXT: v_mov_b32_e32 v18, s10
; VGPR-NEXT: v_mov_b32_e32 v19, s11
; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)
store volatile <16 x float> %arg2, ptr addrspace(1) %out
store volatile <16 x float> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd__flags(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v44, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; SDAG-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; SDAG-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: v_mov_b32_e32 v40, s20
; SDAG-NEXT: v_mov_b32_e32 v41, s21
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-NEXT: v_mov_b32_e32 v42, s22
; SDAG-NEXT: v_mov_b32_e32 v43, s23
; SDAG-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
; SDAG-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
; SDAG-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
; SDAG-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; GISEL-NEXT: v_mov_b32_e32 v56, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:3
; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v44, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; HEURRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; HEURRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: v_mov_b32_e32 v40, s20
; HEURRC-NEXT: v_mov_b32_e32 v41, s21
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; HEURRC-NEXT: v_mov_b32_e32 v42, s22
; HEURRC-NEXT: v_mov_b32_e32 v43, s23
; HEURRC-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
; HEURRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
; HEURRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
; HEURRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v44, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; VGPRRC-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: v_mov_b32_e32 v40, s20
; VGPRRC-NEXT: v_mov_b32_e32 v41, s21
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; VGPRRC-NEXT: v_mov_b32_e32 v42, s22
; VGPRRC-NEXT: v_mov_b32_e32 v43, s23
; VGPRRC-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
; VGPRRC-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; AGPR-NEXT: v_mov_b32_e32 v12, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; AGPR-NEXT: v_accvgpr_write_b32 a31, s23
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; AGPR-NEXT: v_accvgpr_write_b32 a30, s22
; AGPR-NEXT: v_accvgpr_write_b32 a29, s21
; AGPR-NEXT: v_accvgpr_write_b32 a28, s20
; AGPR-NEXT: v_accvgpr_write_b32 a27, s19
; AGPR-NEXT: v_accvgpr_write_b32 a26, s18
; AGPR-NEXT: v_accvgpr_write_b32 a25, s17
; AGPR-NEXT: v_accvgpr_write_b32 a24, s16
; AGPR-NEXT: v_accvgpr_write_b32 a23, s15
; AGPR-NEXT: v_accvgpr_write_b32 a22, s14
; AGPR-NEXT: v_accvgpr_write_b32 a21, s13
; AGPR-NEXT: v_accvgpr_write_b32 a20, s12
; AGPR-NEXT: v_accvgpr_write_b32 a19, s11
; AGPR-NEXT: v_accvgpr_write_b32 a18, s10
; AGPR-NEXT: v_accvgpr_write_b32 a17, s9
; AGPR-NEXT: v_accvgpr_write_b32 a16, s8
; AGPR-NEXT: v_mov_b32_e32 v8, s20
; AGPR-NEXT: v_mov_b32_e32 v9, s21
; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3
; AGPR-NEXT: v_mov_b32_e32 v10, s22
; AGPR-NEXT: v_mov_b32_e32 v11, s23
; AGPR-NEXT: v_mov_b32_e32 v0, s16
; AGPR-NEXT: v_mov_b32_e32 v1, s17
; AGPR-NEXT: v_mov_b32_e32 v2, s18
; AGPR-NEXT: v_mov_b32_e32 v3, s19
; AGPR-NEXT: global_store_dwordx4 v12, v[8:11], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s12
; AGPR-NEXT: v_mov_b32_e32 v1, s13
; AGPR-NEXT: v_mov_b32_e32 v2, s14
; AGPR-NEXT: v_mov_b32_e32 v3, s15
; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, a[8:11], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, a[12:15], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, a[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v12, a[4:7], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPR-NEXT: v_mov_b32_e32 v44, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; VGPR-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; VGPR-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPR-NEXT: v_mov_b32_e32 v40, s20
; VGPR-NEXT: v_mov_b32_e32 v41, s21
; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; VGPR-NEXT: v_mov_b32_e32 v42, s22
; VGPR-NEXT: v_mov_b32_e32 v43, s23
; VGPR-NEXT: global_store_dwordx4 v44, v[40:43], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 2
; VGPR-NEXT: v_mov_b32_e32 v16, s16
; VGPR-NEXT: v_mov_b32_e32 v17, s17
; VGPR-NEXT: v_mov_b32_e32 v18, s18
; VGPR-NEXT: v_mov_b32_e32 v19, s19
; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s12
; VGPR-NEXT: v_mov_b32_e32 v17, s13
; VGPR-NEXT: v_mov_b32_e32 v18, s14
; VGPR-NEXT: v_mov_b32_e32 v19, s15
; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s8
; VGPR-NEXT: v_mov_b32_e32 v17, s9
; VGPR-NEXT: v_mov_b32_e32 v18, s10
; VGPR-NEXT: v_mov_b32_e32 v19, s11
; VGPR-NEXT: global_store_dwordx4 v44, v[16:19], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v44, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v44, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v44, v[0:3], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v44, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 1, i32 2, i32 3)
store volatile <16 x float> %arg2, ptr addrspace(1) %out
store volatile <16 x float> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 7
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]
; GISEL-NEXT: v_mov_b32_e32 v16, 0
; GISEL-NEXT: s_nop 7
; GISEL-NEXT: s_nop 2
; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; HEURRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; HEURRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; HEURRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15]
; AGPR-NEXT: v_mov_b32_e32 v0, 0
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: s_nop 2
; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15]
; VGPR-NEXT: v_mov_b32_e32 v16, 0
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 2
; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 0, i32 0, i32 0)
store <16 x float> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__vgprcd_mac_flags(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 7
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; GISEL-NEXT: v_mov_b32_e32 v16, 0
; GISEL-NEXT: s_nop 7
; GISEL-NEXT: s_nop 2
; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; HEURRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; HEURRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; HEURRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1
; AGPR-NEXT: v_mov_b32_e32 v0, 0
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: s_nop 2
; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_32x32x16_f16__vgprcd_mac_flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_f32_32x32x16_f16 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; VGPR-NEXT: v_mov_b32_e32 v16, 0
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 2
; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; VGPR-NEXT: s_endpgm
%result = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %arg2, i32 3, i32 2, i32 1)
store <16 x float> %result, ptr addrspace(1) %out
ret void
}
; --------------------------------------------------------------------
; llvm.amdgcn.mfma.i32.16x16x64.i8
; --------------------------------------------------------------------
declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32>, <4 x i32>, <4 x i32>, i32 immarg, i32 immarg, i32 immarg)
define <4 x i32> @test_mfma_i32_16x16x64_i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2) {
; GCN-LABEL: test_mfma_i32_16x16x64_i8:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]
; GCN-NEXT: s_nop 7
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_i32_16x16x64_i8:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_i32_16x16x64_i8:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_i32_16x16x64_i8:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 0, i32 0, i32 0)
ret <4 x i32> %result
}
define <4 x i32> @test_mfma_i32_16x16x64_i8__flags(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2) {
; GCN-LABEL: test_mfma_i32_16x16x64_i8__flags:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; GCN-NEXT: s_nop 7
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_i32_16x16x64_i8__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_i32_16x16x64_i8__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_i32_16x16x64_i8__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 1, i32 1, i32 1)
ret <4 x i32> %result
}
define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd(ptr addrspace(1) %out, <4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2) #0 {
; SDAG-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
; SDAG-NEXT: v_mov_b32_e32 v4, s12
; SDAG-NEXT: v_mov_b32_e32 v5, s13
; SDAG-NEXT: v_mov_b32_e32 v6, s14
; SDAG-NEXT: v_mov_b32_e32 v7, s15
; SDAG-NEXT: v_mov_b32_e32 v8, s0
; SDAG-NEXT: v_mov_b32_e32 v9, s1
; SDAG-NEXT: v_mov_b32_e32 v10, s2
; SDAG-NEXT: v_mov_b32_e32 v11, s3
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; SDAG-NEXT: s_nop 7
; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; GISEL-NEXT: v_mov_b32_e32 v4, 0
; GISEL-NEXT: s_nop 6
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
; HEURRC-NEXT: v_mov_b32_e32 v2, s10
; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: v_mov_b32_e32 v4, s12
; HEURRC-NEXT: v_mov_b32_e32 v5, s13
; HEURRC-NEXT: v_mov_b32_e32 v6, s14
; HEURRC-NEXT: v_mov_b32_e32 v7, s15
; HEURRC-NEXT: v_mov_b32_e32 v8, s0
; HEURRC-NEXT: v_mov_b32_e32 v9, s1
; HEURRC-NEXT: v_mov_b32_e32 v10, s2
; HEURRC-NEXT: v_mov_b32_e32 v11, s3
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
; VGPRRC-NEXT: v_mov_b32_e32 v4, s12
; VGPRRC-NEXT: v_mov_b32_e32 v5, s13
; VGPRRC-NEXT: v_mov_b32_e32 v6, s14
; VGPRRC-NEXT: v_mov_b32_e32 v7, s15
; VGPRRC-NEXT: v_mov_b32_e32 v8, s0
; VGPRRC-NEXT: v_mov_b32_e32 v9, s1
; VGPRRC-NEXT: v_mov_b32_e32 v10, s2
; VGPRRC-NEXT: v_mov_b32_e32 v11, s3
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; AGPR-NEXT: v_mov_b32_e32 v8, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: v_mov_b32_e32 v4, s12
; AGPR-NEXT: v_mov_b32_e32 v5, s13
; AGPR-NEXT: v_mov_b32_e32 v6, s14
; AGPR-NEXT: v_mov_b32_e32 v7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a0, s0
; AGPR-NEXT: v_accvgpr_write_b32 a1, s1
; AGPR-NEXT: v_accvgpr_write_b32 a2, s2
; AGPR-NEXT: v_accvgpr_write_b32 a3, s3
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3]
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPR-NEXT: v_mov_b32_e32 v12, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v0, s8
; VGPR-NEXT: v_mov_b32_e32 v1, s9
; VGPR-NEXT: v_mov_b32_e32 v2, s10
; VGPR-NEXT: v_mov_b32_e32 v3, s11
; VGPR-NEXT: v_mov_b32_e32 v4, s12
; VGPR-NEXT: v_mov_b32_e32 v5, s13
; VGPR-NEXT: v_mov_b32_e32 v6, s14
; VGPR-NEXT: v_mov_b32_e32 v7, s15
; VGPR-NEXT: v_mov_b32_e32 v8, s0
; VGPR-NEXT: v_mov_b32_e32 v9, s1
; VGPR-NEXT: v_mov_b32_e32 v10, s2
; VGPR-NEXT: v_mov_b32_e32 v11, s3
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPR-NEXT: s_endpgm
%result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 0, i32 0, i32 0)
store <4 x i32> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags(ptr addrspace(1) %out, <4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2) #0 {
; SDAG-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; SDAG-NEXT: v_mov_b32_e32 v12, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
; SDAG-NEXT: v_mov_b32_e32 v4, s12
; SDAG-NEXT: v_mov_b32_e32 v5, s13
; SDAG-NEXT: v_mov_b32_e32 v6, s14
; SDAG-NEXT: v_mov_b32_e32 v7, s15
; SDAG-NEXT: v_mov_b32_e32 v8, s0
; SDAG-NEXT: v_mov_b32_e32 v9, s1
; SDAG-NEXT: v_mov_b32_e32 v10, s2
; SDAG-NEXT: v_mov_b32_e32 v11, s3
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: s_nop 7
; SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; GISEL-NEXT: v_mov_b32_e32 v4, 0
; GISEL-NEXT: s_nop 6
; GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
; HEURRC-NEXT: v_mov_b32_e32 v2, s10
; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: v_mov_b32_e32 v4, s12
; HEURRC-NEXT: v_mov_b32_e32 v5, s13
; HEURRC-NEXT: v_mov_b32_e32 v6, s14
; HEURRC-NEXT: v_mov_b32_e32 v7, s15
; HEURRC-NEXT: v_mov_b32_e32 v8, s0
; HEURRC-NEXT: v_mov_b32_e32 v9, s1
; HEURRC-NEXT: v_mov_b32_e32 v10, s2
; HEURRC-NEXT: v_mov_b32_e32 v11, s3
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
; VGPRRC-NEXT: v_mov_b32_e32 v4, s12
; VGPRRC-NEXT: v_mov_b32_e32 v5, s13
; VGPRRC-NEXT: v_mov_b32_e32 v6, s14
; VGPRRC-NEXT: v_mov_b32_e32 v7, s15
; VGPRRC-NEXT: v_mov_b32_e32 v8, s0
; VGPRRC-NEXT: v_mov_b32_e32 v9, s1
; VGPRRC-NEXT: v_mov_b32_e32 v10, s2
; VGPRRC-NEXT: v_mov_b32_e32 v11, s3
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; AGPR-NEXT: v_mov_b32_e32 v8, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: v_mov_b32_e32 v4, s12
; AGPR-NEXT: v_mov_b32_e32 v5, s13
; AGPR-NEXT: v_mov_b32_e32 v6, s14
; AGPR-NEXT: v_mov_b32_e32 v7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a0, s0
; AGPR-NEXT: v_accvgpr_write_b32 a1, s1
; AGPR-NEXT: v_accvgpr_write_b32 a2, s2
; AGPR-NEXT: v_accvgpr_write_b32 a3, s3
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_i32_16x16x64_i8_no_agpr__vgprcd__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPR-NEXT: v_mov_b32_e32 v12, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v0, s8
; VGPR-NEXT: v_mov_b32_e32 v1, s9
; VGPR-NEXT: v_mov_b32_e32 v2, s10
; VGPR-NEXT: v_mov_b32_e32 v3, s11
; VGPR-NEXT: v_mov_b32_e32 v4, s12
; VGPR-NEXT: v_mov_b32_e32 v5, s13
; VGPR-NEXT: v_mov_b32_e32 v6, s14
; VGPR-NEXT: v_mov_b32_e32 v7, s15
; VGPR-NEXT: v_mov_b32_e32 v8, s0
; VGPR-NEXT: v_mov_b32_e32 v9, s1
; VGPR-NEXT: v_mov_b32_e32 v10, s2
; VGPR-NEXT: v_mov_b32_e32 v11, s3
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_i32_16x16x64_i8 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPR-NEXT: s_endpgm
%result = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %arg2, i32 3, i32 2, i32 1)
store <4 x i32> %result, ptr addrspace(1) %out
ret void
}
; --------------------------------------------------------------------
; llvm.amdgcn.mfma.i32.32x32x32.i8
; --------------------------------------------------------------------
declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32>, <4 x i32>, <16 x i32>, i32 immarg, i32 immarg, i32 immarg)
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2) #1 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48
; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v4, s24
; SDAG-NEXT: v_mov_b32_e32 v5, s25
; SDAG-NEXT: v_mov_b32_e32 v6, s26
; SDAG-NEXT: v_mov_b32_e32 v7, s27
; SDAG-NEXT: v_accvgpr_write_b32 a31, s23
; SDAG-NEXT: v_mov_b32_e32 v8, s28
; SDAG-NEXT: v_mov_b32_e32 v9, s29
; SDAG-NEXT: v_mov_b32_e32 v10, s30
; SDAG-NEXT: v_mov_b32_e32 v11, s31
; SDAG-NEXT: v_accvgpr_write_b32 a30, s22
; SDAG-NEXT: v_accvgpr_write_b32 a29, s21
; SDAG-NEXT: v_accvgpr_write_b32 a28, s20
; SDAG-NEXT: v_accvgpr_write_b32 a27, s19
; SDAG-NEXT: v_accvgpr_write_b32 a26, s18
; SDAG-NEXT: v_accvgpr_write_b32 a25, s17
; SDAG-NEXT: v_accvgpr_write_b32 a24, s16
; SDAG-NEXT: v_accvgpr_write_b32 a23, s15
; SDAG-NEXT: v_accvgpr_write_b32 a22, s14
; SDAG-NEXT: v_accvgpr_write_b32 a21, s13
; SDAG-NEXT: v_accvgpr_write_b32 a20, s12
; SDAG-NEXT: v_accvgpr_write_b32 a19, s11
; SDAG-NEXT: v_accvgpr_write_b32 a18, s10
; SDAG-NEXT: v_accvgpr_write_b32 a17, s9
; SDAG-NEXT: v_accvgpr_write_b32 a16, s8
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16
; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0
; SDAG-NEXT: v_mov_b32_e32 v8, s16
; SDAG-NEXT: v_mov_b32_e32 v9, s17
; SDAG-NEXT: v_mov_b32_e32 v10, s18
; SDAG-NEXT: v_mov_b32_e32 v11, s19
; SDAG-NEXT: s_nop 5
; SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
; SDAG-NEXT: v_mov_b32_e32 v8, s20
; SDAG-NEXT: v_mov_b32_e32 v9, s21
; SDAG-NEXT: v_mov_b32_e32 v10, s22
; SDAG-NEXT: v_mov_b32_e32 v11, s23
; SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_32x32x32_i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: v_mov_b64_e32 v[12:13], 0
; GISEL-NEXT: v_mov_b64_e32 v[14:15], 16
; GISEL-NEXT: v_mov_b64_e32 v[16:17], 32
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; GISEL-NEXT: v_accvgpr_write_b32 a0, s8
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; GISEL-NEXT: v_accvgpr_write_b32 a1, s9
; GISEL-NEXT: v_accvgpr_write_b32 a2, s10
; GISEL-NEXT: v_accvgpr_write_b32 a3, s11
; GISEL-NEXT: v_accvgpr_write_b32 a4, s12
; GISEL-NEXT: v_accvgpr_write_b32 a5, s13
; GISEL-NEXT: v_accvgpr_write_b32 a6, s14
; GISEL-NEXT: v_accvgpr_write_b32 a7, s15
; GISEL-NEXT: v_accvgpr_write_b32 a8, s16
; GISEL-NEXT: v_accvgpr_write_b32 a9, s17
; GISEL-NEXT: v_accvgpr_write_b32 a10, s18
; GISEL-NEXT: v_accvgpr_write_b32 a11, s19
; GISEL-NEXT: v_accvgpr_write_b32 a12, s20
; GISEL-NEXT: v_accvgpr_write_b32 a13, s21
; GISEL-NEXT: v_accvgpr_write_b32 a14, s22
; GISEL-NEXT: v_accvgpr_write_b32 a15, s23
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[18:19], 48
; GISEL-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
; GISEL-NEXT: s_nop 7
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
; GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
; GISEL-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v4, s24
; HEURRC-NEXT: v_mov_b32_e32 v5, s25
; HEURRC-NEXT: v_mov_b32_e32 v6, s26
; HEURRC-NEXT: v_mov_b32_e32 v7, s27
; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
; HEURRC-NEXT: v_mov_b32_e32 v8, s28
; HEURRC-NEXT: v_mov_b32_e32 v9, s29
; HEURRC-NEXT: v_mov_b32_e32 v10, s30
; HEURRC-NEXT: v_mov_b32_e32 v11, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
; HEURRC-NEXT: v_mov_b32_e32 v9, s17
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
; HEURRC-NEXT: v_mov_b32_e32 v11, s19
; HEURRC-NEXT: s_nop 5
; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v2, s10
; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: v_mov_b32_e32 v8, s20
; HEURRC-NEXT: v_mov_b32_e32 v9, s21
; HEURRC-NEXT: v_mov_b32_e32 v10, s22
; HEURRC-NEXT: v_mov_b32_e32 v11, s23
; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v0, s12
; HEURRC-NEXT: v_mov_b32_e32 v1, s13
; HEURRC-NEXT: v_mov_b32_e32 v2, s14
; HEURRC-NEXT: v_mov_b32_e32 v3, s15
; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 3
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], 16
; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], 0
; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s16
; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b64_e32 v[8:9], 48
; AGPR-NEXT: v_mov_b64_e32 v[10:11], 32
; AGPR-NEXT: v_mov_b64_e32 v[12:13], 16
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s24
; AGPR-NEXT: v_mov_b32_e32 v1, s25
; AGPR-NEXT: v_mov_b32_e32 v2, s26
; AGPR-NEXT: v_mov_b32_e32 v3, s27
; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-NEXT: v_mov_b32_e32 v4, s28
; AGPR-NEXT: v_mov_b32_e32 v5, s29
; AGPR-NEXT: v_mov_b32_e32 v6, s30
; AGPR-NEXT: v_mov_b32_e32 v7, s31
; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-NEXT: v_mov_b64_e32 v[14:15], 0
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15]
; AGPR-NEXT: v_mov_b32_e32 v0, s16
; AGPR-NEXT: v_mov_b32_e32 v1, s17
; AGPR-NEXT: v_mov_b32_e32 v2, s18
; AGPR-NEXT: v_mov_b32_e32 v3, s19
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s12
; AGPR-NEXT: v_mov_b32_e32 v1, s13
; AGPR-NEXT: v_mov_b32_e32 v2, s14
; AGPR-NEXT: v_mov_b32_e32 v3, s15
; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_i32_32x32x32_i8:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b64_e32 v[40:41], 48
; VGPR-NEXT: v_mov_b64_e32 v[42:43], 32
; VGPR-NEXT: v_mov_b64_e32 v[44:45], 16
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v32, s24
; VGPR-NEXT: v_mov_b32_e32 v33, s25
; VGPR-NEXT: v_mov_b32_e32 v34, s26
; VGPR-NEXT: v_mov_b32_e32 v35, s27
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b32_e32 v36, s28
; VGPR-NEXT: v_mov_b32_e32 v37, s29
; VGPR-NEXT: v_mov_b32_e32 v38, s30
; VGPR-NEXT: v_mov_b32_e32 v39, s31
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[46:47], 0
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15]
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 3
; VGPR-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v0, s16
; VGPR-NEXT: v_mov_b32_e32 v1, s17
; VGPR-NEXT: v_mov_b32_e32 v2, s18
; VGPR-NEXT: v_mov_b32_e32 v3, s19
; VGPR-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s20
; VGPR-NEXT: v_mov_b32_e32 v1, s21
; VGPR-NEXT: v_mov_b32_e32 v2, s22
; VGPR-NEXT: v_mov_b32_e32 v3, s23
; VGPR-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s8
; VGPR-NEXT: v_mov_b32_e32 v1, s9
; VGPR-NEXT: v_mov_b32_e32 v2, s10
; VGPR-NEXT: v_mov_b32_e32 v3, s11
; VGPR-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s12
; VGPR-NEXT: v_mov_b32_e32 v1, s13
; VGPR-NEXT: v_mov_b32_e32 v2, s14
; VGPR-NEXT: v_mov_b32_e32 v3, s15
; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0)
store volatile <16 x i32> %result, ptr addrspace(1) null
store volatile <16 x i32> %arg2, ptr addrspace(1) null
ret void
}
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2) #1 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__flags:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b64_e32 v[0:1], 48
; SDAG-NEXT: v_mov_b64_e32 v[2:3], 32
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v4, s24
; SDAG-NEXT: v_mov_b32_e32 v5, s25
; SDAG-NEXT: v_mov_b32_e32 v6, s26
; SDAG-NEXT: v_mov_b32_e32 v7, s27
; SDAG-NEXT: v_accvgpr_write_b32 a31, s23
; SDAG-NEXT: v_mov_b32_e32 v8, s28
; SDAG-NEXT: v_mov_b32_e32 v9, s29
; SDAG-NEXT: v_mov_b32_e32 v10, s30
; SDAG-NEXT: v_mov_b32_e32 v11, s31
; SDAG-NEXT: v_accvgpr_write_b32 a30, s22
; SDAG-NEXT: v_accvgpr_write_b32 a29, s21
; SDAG-NEXT: v_accvgpr_write_b32 a28, s20
; SDAG-NEXT: v_accvgpr_write_b32 a27, s19
; SDAG-NEXT: v_accvgpr_write_b32 a26, s18
; SDAG-NEXT: v_accvgpr_write_b32 a25, s17
; SDAG-NEXT: v_accvgpr_write_b32 a24, s16
; SDAG-NEXT: v_accvgpr_write_b32 a23, s15
; SDAG-NEXT: v_accvgpr_write_b32 a22, s14
; SDAG-NEXT: v_accvgpr_write_b32 a21, s13
; SDAG-NEXT: v_accvgpr_write_b32 a20, s12
; SDAG-NEXT: v_accvgpr_write_b32 a19, s11
; SDAG-NEXT: v_accvgpr_write_b32 a18, s10
; SDAG-NEXT: v_accvgpr_write_b32 a17, s9
; SDAG-NEXT: v_accvgpr_write_b32 a16, s8
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:1
; SDAG-NEXT: v_mov_b64_e32 v[4:5], 16
; SDAG-NEXT: v_mov_b64_e32 v[6:7], 0
; SDAG-NEXT: v_mov_b32_e32 v8, s16
; SDAG-NEXT: v_mov_b32_e32 v9, s17
; SDAG-NEXT: v_mov_b32_e32 v10, s18
; SDAG-NEXT: v_mov_b32_e32 v11, s19
; SDAG-NEXT: s_nop 5
; SDAG-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v2, s10
; SDAG-NEXT: v_mov_b32_e32 v3, s11
; SDAG-NEXT: v_mov_b32_e32 v8, s20
; SDAG-NEXT: v_mov_b32_e32 v9, s21
; SDAG-NEXT: v_mov_b32_e32 v10, s22
; SDAG-NEXT: v_mov_b32_e32 v11, s23
; SDAG-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v0, s8
; SDAG-NEXT: v_mov_b32_e32 v1, s9
; SDAG-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v0, s12
; SDAG-NEXT: v_mov_b32_e32 v1, s13
; SDAG-NEXT: v_mov_b32_e32 v2, s14
; SDAG-NEXT: v_mov_b32_e32 v3, s15
; SDAG-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_32x32x32_i8__flags:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: v_mov_b64_e32 v[12:13], 0
; GISEL-NEXT: v_mov_b64_e32 v[14:15], 16
; GISEL-NEXT: v_mov_b64_e32 v[16:17], 32
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[28:29]
; GISEL-NEXT: v_accvgpr_write_b32 a0, s8
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[30:31]
; GISEL-NEXT: v_accvgpr_write_b32 a1, s9
; GISEL-NEXT: v_accvgpr_write_b32 a2, s10
; GISEL-NEXT: v_accvgpr_write_b32 a3, s11
; GISEL-NEXT: v_accvgpr_write_b32 a4, s12
; GISEL-NEXT: v_accvgpr_write_b32 a5, s13
; GISEL-NEXT: v_accvgpr_write_b32 a6, s14
; GISEL-NEXT: v_accvgpr_write_b32 a7, s15
; GISEL-NEXT: v_accvgpr_write_b32 a8, s16
; GISEL-NEXT: v_accvgpr_write_b32 a9, s17
; GISEL-NEXT: v_accvgpr_write_b32 a10, s18
; GISEL-NEXT: v_accvgpr_write_b32 a11, s19
; GISEL-NEXT: v_accvgpr_write_b32 a12, s20
; GISEL-NEXT: v_accvgpr_write_b32 a13, s21
; GISEL-NEXT: v_accvgpr_write_b32 a14, s22
; GISEL-NEXT: v_accvgpr_write_b32 a15, s23
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[18:19], 48
; GISEL-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
; GISEL-NEXT: s_nop 7
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[18:19]
; GISEL-NEXT: global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_nop 0
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[22:23]
; GISEL-NEXT: global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], 48
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], 32
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v4, s24
; HEURRC-NEXT: v_mov_b32_e32 v5, s25
; HEURRC-NEXT: v_mov_b32_e32 v6, s26
; HEURRC-NEXT: v_mov_b32_e32 v7, s27
; HEURRC-NEXT: v_accvgpr_write_b32 a31, s23
; HEURRC-NEXT: v_mov_b32_e32 v8, s28
; HEURRC-NEXT: v_mov_b32_e32 v9, s29
; HEURRC-NEXT: v_mov_b32_e32 v10, s30
; HEURRC-NEXT: v_mov_b32_e32 v11, s31
; HEURRC-NEXT: v_accvgpr_write_b32 a30, s22
; HEURRC-NEXT: v_accvgpr_write_b32 a29, s21
; HEURRC-NEXT: v_accvgpr_write_b32 a28, s20
; HEURRC-NEXT: v_accvgpr_write_b32 a27, s19
; HEURRC-NEXT: v_accvgpr_write_b32 a26, s18
; HEURRC-NEXT: v_accvgpr_write_b32 a25, s17
; HEURRC-NEXT: v_accvgpr_write_b32 a24, s16
; HEURRC-NEXT: v_accvgpr_write_b32 a23, s15
; HEURRC-NEXT: v_accvgpr_write_b32 a22, s14
; HEURRC-NEXT: v_accvgpr_write_b32 a21, s13
; HEURRC-NEXT: v_accvgpr_write_b32 a20, s12
; HEURRC-NEXT: v_accvgpr_write_b32 a19, s11
; HEURRC-NEXT: v_accvgpr_write_b32 a18, s10
; HEURRC-NEXT: v_accvgpr_write_b32 a17, s9
; HEURRC-NEXT: v_accvgpr_write_b32 a16, s8
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:1
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], 16
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], 0
; HEURRC-NEXT: v_mov_b32_e32 v8, s16
; HEURRC-NEXT: v_mov_b32_e32 v9, s17
; HEURRC-NEXT: v_mov_b32_e32 v10, s18
; HEURRC-NEXT: v_mov_b32_e32 v11, s19
; HEURRC-NEXT: s_nop 5
; HEURRC-NEXT: global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v2, s10
; HEURRC-NEXT: v_mov_b32_e32 v3, s11
; HEURRC-NEXT: v_mov_b32_e32 v8, s20
; HEURRC-NEXT: v_mov_b32_e32 v9, s21
; HEURRC-NEXT: v_mov_b32_e32 v10, s22
; HEURRC-NEXT: v_mov_b32_e32 v11, s23
; HEURRC-NEXT: global_store_dwordx4 v[0:1], v[8:11], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v0, s8
; HEURRC-NEXT: v_mov_b32_e32 v1, s9
; HEURRC-NEXT: global_store_dwordx4 v[6:7], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v0, s12
; HEURRC-NEXT: v_mov_b32_e32 v1, s13
; HEURRC-NEXT: v_mov_b32_e32 v2, s14
; HEURRC-NEXT: v_mov_b32_e32 v3, s15
; HEURRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b64_e32 v[32:33], 48
; VGPRRC-NEXT: v_mov_b64_e32 v[34:35], 32
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b32_e32 v40, s28
; VGPRRC-NEXT: v_mov_b32_e32 v41, s29
; VGPRRC-NEXT: v_mov_b32_e32 v42, s30
; VGPRRC-NEXT: v_mov_b32_e32 v43, s31
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 3
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], 16
; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[4:7], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], 0
; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s16
; VGPRRC-NEXT: v_mov_b32_e32 v1, s17
; VGPRRC-NEXT: v_mov_b32_e32 v2, s18
; VGPRRC-NEXT: v_mov_b32_e32 v3, s19
; VGPRRC-NEXT: global_store_dwordx4 v[34:35], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s20
; VGPRRC-NEXT: v_mov_b32_e32 v1, s21
; VGPRRC-NEXT: v_mov_b32_e32 v2, s22
; VGPRRC-NEXT: v_mov_b32_e32 v3, s23
; VGPRRC-NEXT: global_store_dwordx4 v[32:33], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s8
; VGPRRC-NEXT: v_mov_b32_e32 v1, s9
; VGPRRC-NEXT: v_mov_b32_e32 v2, s10
; VGPRRC-NEXT: v_mov_b32_e32 v3, s11
; VGPRRC-NEXT: global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v0, s12
; VGPRRC-NEXT: v_mov_b32_e32 v1, s13
; VGPRRC-NEXT: v_mov_b32_e32 v2, s14
; VGPRRC-NEXT: v_mov_b32_e32 v3, s15
; VGPRRC-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b64_e32 v[8:9], 48
; AGPR-NEXT: v_mov_b64_e32 v[10:11], 32
; AGPR-NEXT: v_mov_b64_e32 v[12:13], 16
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s24
; AGPR-NEXT: v_mov_b32_e32 v1, s25
; AGPR-NEXT: v_mov_b32_e32 v2, s26
; AGPR-NEXT: v_mov_b32_e32 v3, s27
; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-NEXT: v_mov_b32_e32 v4, s28
; AGPR-NEXT: v_mov_b32_e32 v5, s29
; AGPR-NEXT: v_mov_b32_e32 v6, s30
; AGPR-NEXT: v_mov_b32_e32 v7, s31
; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-NEXT: v_mov_b64_e32 v[14:15], 0
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
; AGPR-NEXT: v_mov_b32_e32 v0, s16
; AGPR-NEXT: v_mov_b32_e32 v1, s17
; AGPR-NEXT: v_mov_b32_e32 v2, s18
; AGPR-NEXT: v_mov_b32_e32 v3, s19
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[10:11], a[24:27], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[12:13], a[20:23], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[14:15], a[16:19], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v[10:11], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s12
; AGPR-NEXT: v_mov_b32_e32 v1, s13
; AGPR-NEXT: v_mov_b32_e32 v2, s14
; AGPR-NEXT: v_mov_b32_e32 v3, s15
; AGPR-NEXT: global_store_dwordx4 v[12:13], v[0:3], off sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_i32_32x32x32_i8__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b64_e32 v[40:41], 48
; VGPR-NEXT: v_mov_b64_e32 v[42:43], 32
; VGPR-NEXT: v_mov_b64_e32 v[44:45], 16
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v32, s24
; VGPR-NEXT: v_mov_b32_e32 v33, s25
; VGPR-NEXT: v_mov_b32_e32 v34, s26
; VGPR-NEXT: v_mov_b32_e32 v35, s27
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b32_e32 v36, s28
; VGPR-NEXT: v_mov_b32_e32 v37, s29
; VGPR-NEXT: v_mov_b32_e32 v38, s30
; VGPR-NEXT: v_mov_b32_e32 v39, s31
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[46:47], 0
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 3
; VGPR-NEXT: global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[42:43], v[24:27], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[44:45], v[20:23], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v0, s16
; VGPR-NEXT: v_mov_b32_e32 v1, s17
; VGPR-NEXT: v_mov_b32_e32 v2, s18
; VGPR-NEXT: v_mov_b32_e32 v3, s19
; VGPR-NEXT: global_store_dwordx4 v[42:43], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s20
; VGPR-NEXT: v_mov_b32_e32 v1, s21
; VGPR-NEXT: v_mov_b32_e32 v2, s22
; VGPR-NEXT: v_mov_b32_e32 v3, s23
; VGPR-NEXT: global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s8
; VGPR-NEXT: v_mov_b32_e32 v1, s9
; VGPR-NEXT: v_mov_b32_e32 v2, s10
; VGPR-NEXT: v_mov_b32_e32 v3, s11
; VGPR-NEXT: global_store_dwordx4 v[46:47], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v0, s12
; VGPR-NEXT: v_mov_b32_e32 v1, s13
; VGPR-NEXT: v_mov_b32_e32 v2, s14
; VGPR-NEXT: v_mov_b32_e32 v3, s15
; VGPR-NEXT: global_store_dwordx4 v[44:45], v[0:3], off sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 2, i32 3, i32 1)
store volatile <16 x i32> %result, ptr addrspace(1) null
store volatile <16 x i32> %arg2, ptr addrspace(1) null
ret void
}
define <16 x i32> @test_mfma_i32_32x32x32_i8__mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2) {
; GCN-LABEL: test_mfma_i32_32x32x32_i8__mac:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: v_accvgpr_write_b32 a4, v12
; GCN-NEXT: v_accvgpr_write_b32 a5, v13
; GCN-NEXT: v_accvgpr_write_b32 a6, v14
; GCN-NEXT: v_accvgpr_write_b32 a7, v15
; GCN-NEXT: v_accvgpr_write_b32 a8, v16
; GCN-NEXT: v_accvgpr_write_b32 a9, v17
; GCN-NEXT: v_accvgpr_write_b32 a10, v18
; GCN-NEXT: v_accvgpr_write_b32 a11, v19
; GCN-NEXT: v_accvgpr_write_b32 a12, v20
; GCN-NEXT: v_accvgpr_write_b32 a13, v21
; GCN-NEXT: v_accvgpr_write_b32 a14, v22
; GCN-NEXT: v_accvgpr_write_b32 a15, v23
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]
; GCN-NEXT: s_nop 7
; GCN-NEXT: s_nop 3
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: v_accvgpr_read_b32 v4, a4
; GCN-NEXT: v_accvgpr_read_b32 v5, a5
; GCN-NEXT: v_accvgpr_read_b32 v6, a6
; GCN-NEXT: v_accvgpr_read_b32 v7, a7
; GCN-NEXT: v_accvgpr_read_b32 v8, a8
; GCN-NEXT: v_accvgpr_read_b32 v9, a9
; GCN-NEXT: v_accvgpr_read_b32 v10, a10
; GCN-NEXT: v_accvgpr_read_b32 v11, a11
; GCN-NEXT: v_accvgpr_read_b32 v12, a12
; GCN-NEXT: v_accvgpr_read_b32 v13, a13
; GCN-NEXT: v_accvgpr_read_b32 v14, a14
; GCN-NEXT: v_accvgpr_read_b32 v15, a15
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__mac:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: v_accvgpr_write_b32 a4, v12
; HEURRC-NEXT: v_accvgpr_write_b32 a5, v13
; HEURRC-NEXT: v_accvgpr_write_b32 a6, v14
; HEURRC-NEXT: v_accvgpr_write_b32 a7, v15
; HEURRC-NEXT: v_accvgpr_write_b32 a8, v16
; HEURRC-NEXT: v_accvgpr_write_b32 a9, v17
; HEURRC-NEXT: v_accvgpr_write_b32 a10, v18
; HEURRC-NEXT: v_accvgpr_write_b32 a11, v19
; HEURRC-NEXT: v_accvgpr_write_b32 a12, v20
; HEURRC-NEXT: v_accvgpr_write_b32 a13, v21
; HEURRC-NEXT: v_accvgpr_write_b32 a14, v22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, v23
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: s_nop 3
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: v_accvgpr_read_b32 v4, a4
; HEURRC-NEXT: v_accvgpr_read_b32 v5, a5
; HEURRC-NEXT: v_accvgpr_read_b32 v6, a6
; HEURRC-NEXT: v_accvgpr_read_b32 v7, a7
; HEURRC-NEXT: v_accvgpr_read_b32 v8, a8
; HEURRC-NEXT: v_accvgpr_read_b32 v9, a9
; HEURRC-NEXT: v_accvgpr_read_b32 v10, a10
; HEURRC-NEXT: v_accvgpr_read_b32 v11, a11
; HEURRC-NEXT: v_accvgpr_read_b32 v12, a12
; HEURRC-NEXT: v_accvgpr_read_b32 v13, a13
; HEURRC-NEXT: v_accvgpr_read_b32 v14, a14
; HEURRC-NEXT: v_accvgpr_read_b32 v15, a15
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__mac:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23]
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 3
; VGPRRC-NEXT: v_mov_b32_e32 v0, v8
; VGPRRC-NEXT: v_mov_b32_e32 v1, v9
; VGPRRC-NEXT: v_mov_b32_e32 v2, v10
; VGPRRC-NEXT: v_mov_b32_e32 v3, v11
; VGPRRC-NEXT: v_mov_b32_e32 v4, v12
; VGPRRC-NEXT: v_mov_b32_e32 v5, v13
; VGPRRC-NEXT: v_mov_b32_e32 v6, v14
; VGPRRC-NEXT: v_mov_b32_e32 v7, v15
; VGPRRC-NEXT: v_mov_b32_e32 v8, v16
; VGPRRC-NEXT: v_mov_b32_e32 v9, v17
; VGPRRC-NEXT: v_mov_b32_e32 v10, v18
; VGPRRC-NEXT: v_mov_b32_e32 v11, v19
; VGPRRC-NEXT: v_mov_b32_e32 v12, v20
; VGPRRC-NEXT: v_mov_b32_e32 v13, v21
; VGPRRC-NEXT: v_mov_b32_e32 v14, v22
; VGPRRC-NEXT: v_mov_b32_e32 v15, v23
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__mac:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: v_accvgpr_write_b32 a4, v12
; AGPR-NEXT: v_accvgpr_write_b32 a5, v13
; AGPR-NEXT: v_accvgpr_write_b32 a6, v14
; AGPR-NEXT: v_accvgpr_write_b32 a7, v15
; AGPR-NEXT: v_accvgpr_write_b32 a8, v16
; AGPR-NEXT: v_accvgpr_write_b32 a9, v17
; AGPR-NEXT: v_accvgpr_write_b32 a10, v18
; AGPR-NEXT: v_accvgpr_write_b32 a11, v19
; AGPR-NEXT: v_accvgpr_write_b32 a12, v20
; AGPR-NEXT: v_accvgpr_write_b32 a13, v21
; AGPR-NEXT: v_accvgpr_write_b32 a14, v22
; AGPR-NEXT: v_accvgpr_write_b32 a15, v23
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: s_nop 3
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: v_accvgpr_read_b32 v4, a4
; AGPR-NEXT: v_accvgpr_read_b32 v5, a5
; AGPR-NEXT: v_accvgpr_read_b32 v6, a6
; AGPR-NEXT: v_accvgpr_read_b32 v7, a7
; AGPR-NEXT: v_accvgpr_read_b32 v8, a8
; AGPR-NEXT: v_accvgpr_read_b32 v9, a9
; AGPR-NEXT: v_accvgpr_read_b32 v10, a10
; AGPR-NEXT: v_accvgpr_read_b32 v11, a11
; AGPR-NEXT: v_accvgpr_read_b32 v12, a12
; AGPR-NEXT: v_accvgpr_read_b32 v13, a13
; AGPR-NEXT: v_accvgpr_read_b32 v14, a14
; AGPR-NEXT: v_accvgpr_read_b32 v15, a15
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_i32_32x32x32_i8__mac:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23]
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 3
; VGPR-NEXT: v_mov_b32_e32 v0, v8
; VGPR-NEXT: v_mov_b32_e32 v1, v9
; VGPR-NEXT: v_mov_b32_e32 v2, v10
; VGPR-NEXT: v_mov_b32_e32 v3, v11
; VGPR-NEXT: v_mov_b32_e32 v4, v12
; VGPR-NEXT: v_mov_b32_e32 v5, v13
; VGPR-NEXT: v_mov_b32_e32 v6, v14
; VGPR-NEXT: v_mov_b32_e32 v7, v15
; VGPR-NEXT: v_mov_b32_e32 v8, v16
; VGPR-NEXT: v_mov_b32_e32 v9, v17
; VGPR-NEXT: v_mov_b32_e32 v10, v18
; VGPR-NEXT: v_mov_b32_e32 v11, v19
; VGPR-NEXT: v_mov_b32_e32 v12, v20
; VGPR-NEXT: v_mov_b32_e32 v13, v21
; VGPR-NEXT: v_mov_b32_e32 v14, v22
; VGPR-NEXT: v_mov_b32_e32 v15, v23
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0)
ret <16 x i32> %result
}
define <16 x i32> @test_mfma_i32_32x32x32_i8__mac__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2) {
; GCN-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: v_accvgpr_write_b32 a4, v12
; GCN-NEXT: v_accvgpr_write_b32 a5, v13
; GCN-NEXT: v_accvgpr_write_b32 a6, v14
; GCN-NEXT: v_accvgpr_write_b32 a7, v15
; GCN-NEXT: v_accvgpr_write_b32 a8, v16
; GCN-NEXT: v_accvgpr_write_b32 a9, v17
; GCN-NEXT: v_accvgpr_write_b32 a10, v18
; GCN-NEXT: v_accvgpr_write_b32 a11, v19
; GCN-NEXT: v_accvgpr_write_b32 a12, v20
; GCN-NEXT: v_accvgpr_write_b32 a13, v21
; GCN-NEXT: v_accvgpr_write_b32 a14, v22
; GCN-NEXT: v_accvgpr_write_b32 a15, v23
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
; GCN-NEXT: s_nop 7
; GCN-NEXT: s_nop 3
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: v_accvgpr_read_b32 v4, a4
; GCN-NEXT: v_accvgpr_read_b32 v5, a5
; GCN-NEXT: v_accvgpr_read_b32 v6, a6
; GCN-NEXT: v_accvgpr_read_b32 v7, a7
; GCN-NEXT: v_accvgpr_read_b32 v8, a8
; GCN-NEXT: v_accvgpr_read_b32 v9, a9
; GCN-NEXT: v_accvgpr_read_b32 v10, a10
; GCN-NEXT: v_accvgpr_read_b32 v11, a11
; GCN-NEXT: v_accvgpr_read_b32 v12, a12
; GCN-NEXT: v_accvgpr_read_b32 v13, a13
; GCN-NEXT: v_accvgpr_read_b32 v14, a14
; GCN-NEXT: v_accvgpr_read_b32 v15, a15
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: v_accvgpr_write_b32 a4, v12
; HEURRC-NEXT: v_accvgpr_write_b32 a5, v13
; HEURRC-NEXT: v_accvgpr_write_b32 a6, v14
; HEURRC-NEXT: v_accvgpr_write_b32 a7, v15
; HEURRC-NEXT: v_accvgpr_write_b32 a8, v16
; HEURRC-NEXT: v_accvgpr_write_b32 a9, v17
; HEURRC-NEXT: v_accvgpr_write_b32 a10, v18
; HEURRC-NEXT: v_accvgpr_write_b32 a11, v19
; HEURRC-NEXT: v_accvgpr_write_b32 a12, v20
; HEURRC-NEXT: v_accvgpr_write_b32 a13, v21
; HEURRC-NEXT: v_accvgpr_write_b32 a14, v22
; HEURRC-NEXT: v_accvgpr_write_b32 a15, v23
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: s_nop 3
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: v_accvgpr_read_b32 v4, a4
; HEURRC-NEXT: v_accvgpr_read_b32 v5, a5
; HEURRC-NEXT: v_accvgpr_read_b32 v6, a6
; HEURRC-NEXT: v_accvgpr_read_b32 v7, a7
; HEURRC-NEXT: v_accvgpr_read_b32 v8, a8
; HEURRC-NEXT: v_accvgpr_read_b32 v9, a9
; HEURRC-NEXT: v_accvgpr_read_b32 v10, a10
; HEURRC-NEXT: v_accvgpr_read_b32 v11, a11
; HEURRC-NEXT: v_accvgpr_read_b32 v12, a12
; HEURRC-NEXT: v_accvgpr_read_b32 v13, a13
; HEURRC-NEXT: v_accvgpr_read_b32 v14, a14
; HEURRC-NEXT: v_accvgpr_read_b32 v15, a15
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:1
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 3
; VGPRRC-NEXT: v_mov_b32_e32 v0, v8
; VGPRRC-NEXT: v_mov_b32_e32 v1, v9
; VGPRRC-NEXT: v_mov_b32_e32 v2, v10
; VGPRRC-NEXT: v_mov_b32_e32 v3, v11
; VGPRRC-NEXT: v_mov_b32_e32 v4, v12
; VGPRRC-NEXT: v_mov_b32_e32 v5, v13
; VGPRRC-NEXT: v_mov_b32_e32 v6, v14
; VGPRRC-NEXT: v_mov_b32_e32 v7, v15
; VGPRRC-NEXT: v_mov_b32_e32 v8, v16
; VGPRRC-NEXT: v_mov_b32_e32 v9, v17
; VGPRRC-NEXT: v_mov_b32_e32 v10, v18
; VGPRRC-NEXT: v_mov_b32_e32 v11, v19
; VGPRRC-NEXT: v_mov_b32_e32 v12, v20
; VGPRRC-NEXT: v_mov_b32_e32 v13, v21
; VGPRRC-NEXT: v_mov_b32_e32 v14, v22
; VGPRRC-NEXT: v_mov_b32_e32 v15, v23
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: v_accvgpr_write_b32 a4, v12
; AGPR-NEXT: v_accvgpr_write_b32 a5, v13
; AGPR-NEXT: v_accvgpr_write_b32 a6, v14
; AGPR-NEXT: v_accvgpr_write_b32 a7, v15
; AGPR-NEXT: v_accvgpr_write_b32 a8, v16
; AGPR-NEXT: v_accvgpr_write_b32 a9, v17
; AGPR-NEXT: v_accvgpr_write_b32 a10, v18
; AGPR-NEXT: v_accvgpr_write_b32 a11, v19
; AGPR-NEXT: v_accvgpr_write_b32 a12, v20
; AGPR-NEXT: v_accvgpr_write_b32 a13, v21
; AGPR-NEXT: v_accvgpr_write_b32 a14, v22
; AGPR-NEXT: v_accvgpr_write_b32 a15, v23
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:1 abid:1 blgp:1
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: s_nop 3
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: v_accvgpr_read_b32 v4, a4
; AGPR-NEXT: v_accvgpr_read_b32 v5, a5
; AGPR-NEXT: v_accvgpr_read_b32 v6, a6
; AGPR-NEXT: v_accvgpr_read_b32 v7, a7
; AGPR-NEXT: v_accvgpr_read_b32 v8, a8
; AGPR-NEXT: v_accvgpr_read_b32 v9, a9
; AGPR-NEXT: v_accvgpr_read_b32 v10, a10
; AGPR-NEXT: v_accvgpr_read_b32 v11, a11
; AGPR-NEXT: v_accvgpr_read_b32 v12, a12
; AGPR-NEXT: v_accvgpr_read_b32 v13, a13
; AGPR-NEXT: v_accvgpr_read_b32 v14, a14
; AGPR-NEXT: v_accvgpr_read_b32 v15, a15
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_i32_32x32x32_i8__mac__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[8:23], v[0:3], v[4:7], v[8:23] cbsz:1 abid:1 blgp:1
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 3
; VGPR-NEXT: v_mov_b32_e32 v0, v8
; VGPR-NEXT: v_mov_b32_e32 v1, v9
; VGPR-NEXT: v_mov_b32_e32 v2, v10
; VGPR-NEXT: v_mov_b32_e32 v3, v11
; VGPR-NEXT: v_mov_b32_e32 v4, v12
; VGPR-NEXT: v_mov_b32_e32 v5, v13
; VGPR-NEXT: v_mov_b32_e32 v6, v14
; VGPR-NEXT: v_mov_b32_e32 v7, v15
; VGPR-NEXT: v_mov_b32_e32 v8, v16
; VGPR-NEXT: v_mov_b32_e32 v9, v17
; VGPR-NEXT: v_mov_b32_e32 v10, v18
; VGPR-NEXT: v_mov_b32_e32 v11, v19
; VGPR-NEXT: v_mov_b32_e32 v12, v20
; VGPR-NEXT: v_mov_b32_e32 v13, v21
; VGPR-NEXT: v_mov_b32_e32 v14, v22
; VGPR-NEXT: v_mov_b32_e32 v15, v23
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 1, i32 1, i32 1)
ret <16 x i32> %result
}
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v32, s20
; SDAG-NEXT: v_mov_b32_e32 v33, s21
; SDAG-NEXT: v_mov_b32_e32 v34, s22
; SDAG-NEXT: v_mov_b32_e32 v35, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v36, s24
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
; SDAG-NEXT: v_mov_b32_e32 v19, s23
; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; GISEL-NEXT: v_mov_b32_e32 v56, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15]
; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v32, s20
; HEURRC-NEXT: v_mov_b32_e32 v33, s21
; HEURRC-NEXT: v_mov_b32_e32 v34, s22
; HEURRC-NEXT: v_mov_b32_e32 v35, s23
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v36, s24
; HEURRC-NEXT: v_mov_b32_e32 v37, s25
; HEURRC-NEXT: v_mov_b32_e32 v38, s26
; HEURRC-NEXT: v_mov_b32_e32 v39, s27
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
; HEURRC-NEXT: v_mov_b32_e32 v18, s22
; HEURRC-NEXT: v_mov_b32_e32 v19, s23
; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; AGPR-NEXT: v_mov_b32_e32 v8, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b32_e32 v4, s24
; AGPR-NEXT: v_mov_b32_e32 v5, s25
; AGPR-NEXT: v_mov_b32_e32 v6, s26
; AGPR-NEXT: v_mov_b32_e32 v7, s27
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a31, s23
; AGPR-NEXT: v_accvgpr_write_b32 a30, s22
; AGPR-NEXT: v_accvgpr_write_b32 a29, s21
; AGPR-NEXT: v_accvgpr_write_b32 a28, s20
; AGPR-NEXT: v_accvgpr_write_b32 a27, s19
; AGPR-NEXT: v_accvgpr_write_b32 a26, s18
; AGPR-NEXT: v_accvgpr_write_b32 a25, s17
; AGPR-NEXT: v_accvgpr_write_b32 a24, s16
; AGPR-NEXT: v_accvgpr_write_b32 a23, s15
; AGPR-NEXT: v_accvgpr_write_b32 a22, s14
; AGPR-NEXT: v_accvgpr_write_b32 a21, s13
; AGPR-NEXT: v_accvgpr_write_b32 a20, s12
; AGPR-NEXT: v_accvgpr_write_b32 a19, s11
; AGPR-NEXT: v_accvgpr_write_b32 a18, s10
; AGPR-NEXT: v_accvgpr_write_b32 a17, s9
; AGPR-NEXT: v_accvgpr_write_b32 a16, s8
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31]
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s16
; AGPR-NEXT: v_mov_b32_e32 v1, s17
; AGPR-NEXT: v_mov_b32_e32 v2, s18
; AGPR-NEXT: v_mov_b32_e32 v3, s19
; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s12
; AGPR-NEXT: v_mov_b32_e32 v1, s13
; AGPR-NEXT: v_mov_b32_e32 v2, s14
; AGPR-NEXT: v_mov_b32_e32 v3, s15
; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPR-NEXT: v_mov_b32_e32 v40, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v32, s20
; VGPR-NEXT: v_mov_b32_e32 v33, s21
; VGPR-NEXT: v_mov_b32_e32 v34, s22
; VGPR-NEXT: v_mov_b32_e32 v35, s23
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b32_e32 v36, s24
; VGPR-NEXT: v_mov_b32_e32 v37, s25
; VGPR-NEXT: v_mov_b32_e32 v38, s26
; VGPR-NEXT: v_mov_b32_e32 v39, s27
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31]
; VGPR-NEXT: s_nop 6
; VGPR-NEXT: v_mov_b32_e32 v16, s20
; VGPR-NEXT: v_mov_b32_e32 v17, s21
; VGPR-NEXT: v_mov_b32_e32 v18, s22
; VGPR-NEXT: v_mov_b32_e32 v19, s23
; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s16
; VGPR-NEXT: v_mov_b32_e32 v17, s17
; VGPR-NEXT: v_mov_b32_e32 v18, s18
; VGPR-NEXT: v_mov_b32_e32 v19, s19
; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s12
; VGPR-NEXT: v_mov_b32_e32 v17, s13
; VGPR-NEXT: v_mov_b32_e32 v18, s14
; VGPR-NEXT: v_mov_b32_e32 v19, s15
; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s8
; VGPR-NEXT: v_mov_b32_e32 v17, s9
; VGPR-NEXT: v_mov_b32_e32 v18, s10
; VGPR-NEXT: v_mov_b32_e32 v19, s11
; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0)
store volatile <16 x i32> %arg2, ptr addrspace(1) %out
store volatile <16 x i32> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd__flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: v_mov_b32_e32 v40, 0
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v32, s20
; SDAG-NEXT: v_mov_b32_e32 v33, s21
; SDAG-NEXT: v_mov_b32_e32 v34, s22
; SDAG-NEXT: v_mov_b32_e32 v35, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v36, s24
; SDAG-NEXT: v_mov_b32_e32 v37, s25
; SDAG-NEXT: v_mov_b32_e32 v38, s26
; SDAG-NEXT: v_mov_b32_e32 v39, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; SDAG-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; SDAG-NEXT: s_nop 6
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
; SDAG-NEXT: v_mov_b32_e32 v19, s23
; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s16
; SDAG-NEXT: v_mov_b32_e32 v17, s17
; SDAG-NEXT: v_mov_b32_e32 v18, s18
; SDAG-NEXT: v_mov_b32_e32 v19, s19
; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s12
; SDAG-NEXT: v_mov_b32_e32 v17, s13
; SDAG-NEXT: v_mov_b32_e32 v18, s14
; SDAG-NEXT: v_mov_b32_e32 v19, s15
; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_nop 0
; SDAG-NEXT: v_mov_b32_e32 v16, s8
; SDAG-NEXT: v_mov_b32_e32 v17, s9
; SDAG-NEXT: v_mov_b32_e32 v18, s10
; SDAG-NEXT: v_mov_b32_e32 v19, s11
; SDAG-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; GISEL-NEXT: v_mov_b32_e32 v56, 0
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[34:35], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[32:33], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[38:39], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[36:37], s[28:29]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[42:43], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[40:41], s[8:9]
; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:1 abid:2 blgp:3
; GISEL-NEXT: v_mov_b64_e32 v[46:47], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[50:51], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[54:55], s[22:23]
; GISEL-NEXT: v_mov_b64_e32 v[44:45], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[48:49], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[52:53], s[20:21]
; GISEL-NEXT: global_store_dwordx4 v56, v[40:43], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[44:47], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[48:51], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[52:55], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[16:19], s[0:1] sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[20:23], s[0:1] offset:16 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[24:27], s[0:1] offset:32 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: global_store_dwordx4 v56, v[28:31], s[0:1] offset:48 sc0 sc1
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: v_mov_b32_e32 v40, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v32, s20
; HEURRC-NEXT: v_mov_b32_e32 v33, s21
; HEURRC-NEXT: v_mov_b32_e32 v34, s22
; HEURRC-NEXT: v_mov_b32_e32 v35, s23
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v36, s24
; HEURRC-NEXT: v_mov_b32_e32 v37, s25
; HEURRC-NEXT: v_mov_b32_e32 v38, s26
; HEURRC-NEXT: v_mov_b32_e32 v39, s27
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; HEURRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; HEURRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; HEURRC-NEXT: s_nop 6
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
; HEURRC-NEXT: v_mov_b32_e32 v18, s22
; HEURRC-NEXT: v_mov_b32_e32 v19, s23
; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s16
; HEURRC-NEXT: v_mov_b32_e32 v17, s17
; HEURRC-NEXT: v_mov_b32_e32 v18, s18
; HEURRC-NEXT: v_mov_b32_e32 v19, s19
; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s12
; HEURRC-NEXT: v_mov_b32_e32 v17, s13
; HEURRC-NEXT: v_mov_b32_e32 v18, s14
; HEURRC-NEXT: v_mov_b32_e32 v19, s15
; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_nop 0
; HEURRC-NEXT: v_mov_b32_e32 v16, s8
; HEURRC-NEXT: v_mov_b32_e32 v17, s9
; HEURRC-NEXT: v_mov_b32_e32 v18, s10
; HEURRC-NEXT: v_mov_b32_e32 v19, s11
; HEURRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; HEURRC-NEXT: s_waitcnt vmcnt(0)
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: v_mov_b32_e32 v40, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v32, s20
; VGPRRC-NEXT: v_mov_b32_e32 v33, s21
; VGPRRC-NEXT: v_mov_b32_e32 v34, s22
; VGPRRC-NEXT: v_mov_b32_e32 v35, s23
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v36, s24
; VGPRRC-NEXT: v_mov_b32_e32 v37, s25
; VGPRRC-NEXT: v_mov_b32_e32 v38, s26
; VGPRRC-NEXT: v_mov_b32_e32 v39, s27
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPRRC-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; VGPRRC-NEXT: s_nop 6
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s16
; VGPRRC-NEXT: v_mov_b32_e32 v17, s17
; VGPRRC-NEXT: v_mov_b32_e32 v18, s18
; VGPRRC-NEXT: v_mov_b32_e32 v19, s19
; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s12
; VGPRRC-NEXT: v_mov_b32_e32 v17, s13
; VGPRRC-NEXT: v_mov_b32_e32 v18, s14
; VGPRRC-NEXT: v_mov_b32_e32 v19, s15
; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_nop 0
; VGPRRC-NEXT: v_mov_b32_e32 v16, s8
; VGPRRC-NEXT: v_mov_b32_e32 v17, s9
; VGPRRC-NEXT: v_mov_b32_e32 v18, s10
; VGPRRC-NEXT: v_mov_b32_e32 v19, s11
; VGPRRC-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPRRC-NEXT: s_waitcnt vmcnt(0)
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; AGPR-NEXT: v_mov_b32_e32 v8, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b32_e32 v4, s24
; AGPR-NEXT: v_mov_b32_e32 v5, s25
; AGPR-NEXT: v_mov_b32_e32 v6, s26
; AGPR-NEXT: v_mov_b32_e32 v7, s27
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a31, s23
; AGPR-NEXT: v_accvgpr_write_b32 a30, s22
; AGPR-NEXT: v_accvgpr_write_b32 a29, s21
; AGPR-NEXT: v_accvgpr_write_b32 a28, s20
; AGPR-NEXT: v_accvgpr_write_b32 a27, s19
; AGPR-NEXT: v_accvgpr_write_b32 a26, s18
; AGPR-NEXT: v_accvgpr_write_b32 a25, s17
; AGPR-NEXT: v_accvgpr_write_b32 a24, s16
; AGPR-NEXT: v_accvgpr_write_b32 a23, s15
; AGPR-NEXT: v_accvgpr_write_b32 a22, s14
; AGPR-NEXT: v_accvgpr_write_b32 a21, s13
; AGPR-NEXT: v_accvgpr_write_b32 a20, s12
; AGPR-NEXT: v_accvgpr_write_b32 a19, s11
; AGPR-NEXT: v_accvgpr_write_b32 a18, s10
; AGPR-NEXT: v_accvgpr_write_b32 a17, s9
; AGPR-NEXT: v_accvgpr_write_b32 a16, s8
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[16:31] cbsz:1 abid:2 blgp:3
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s16
; AGPR-NEXT: v_mov_b32_e32 v1, s17
; AGPR-NEXT: v_mov_b32_e32 v2, s18
; AGPR-NEXT: v_mov_b32_e32 v3, s19
; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s12
; AGPR-NEXT: v_mov_b32_e32 v1, s13
; AGPR-NEXT: v_mov_b32_e32 v2, s14
; AGPR-NEXT: v_mov_b32_e32 v3, s15
; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_nop 0
; AGPR-NEXT: v_mov_b32_e32 v0, s8
; AGPR-NEXT: v_mov_b32_e32 v1, s9
; AGPR-NEXT: v_mov_b32_e32 v2, s10
; AGPR-NEXT: v_mov_b32_e32 v3, s11
; AGPR-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v8, a[8:11], s[0:1] offset:32 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v8, a[12:15], s[0:1] offset:48 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[0:1] sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: global_store_dwordx4 v8, a[4:7], s[0:1] offset:16 sc0 sc1
; AGPR-NEXT: s_waitcnt vmcnt(0)
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPR-NEXT: v_mov_b32_e32 v40, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v32, s20
; VGPR-NEXT: v_mov_b32_e32 v33, s21
; VGPR-NEXT: v_mov_b32_e32 v34, s22
; VGPR-NEXT: v_mov_b32_e32 v35, s23
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b32_e32 v36, s24
; VGPR-NEXT: v_mov_b32_e32 v37, s25
; VGPR-NEXT: v_mov_b32_e32 v38, s26
; VGPR-NEXT: v_mov_b32_e32 v39, s27
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[30:31], s[22:23]
; VGPR-NEXT: v_mov_b64_e32 v[28:29], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[26:27], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[24:25], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[22:23], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[20:21], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[18:19], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[16:17], s[8:9]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[32:35], v[36:39], v[16:31] cbsz:1 abid:2 blgp:3
; VGPR-NEXT: s_nop 6
; VGPR-NEXT: v_mov_b32_e32 v16, s20
; VGPR-NEXT: v_mov_b32_e32 v17, s21
; VGPR-NEXT: v_mov_b32_e32 v18, s22
; VGPR-NEXT: v_mov_b32_e32 v19, s23
; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s16
; VGPR-NEXT: v_mov_b32_e32 v17, s17
; VGPR-NEXT: v_mov_b32_e32 v18, s18
; VGPR-NEXT: v_mov_b32_e32 v19, s19
; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s12
; VGPR-NEXT: v_mov_b32_e32 v17, s13
; VGPR-NEXT: v_mov_b32_e32 v18, s14
; VGPR-NEXT: v_mov_b32_e32 v19, s15
; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_nop 0
; VGPR-NEXT: v_mov_b32_e32 v16, s8
; VGPR-NEXT: v_mov_b32_e32 v17, s9
; VGPR-NEXT: v_mov_b32_e32 v18, s10
; VGPR-NEXT: v_mov_b32_e32 v19, s11
; VGPR-NEXT: global_store_dwordx4 v40, v[16:19], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v40, v[8:11], s[0:1] offset:32 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v40, v[12:15], s[0:1] offset:48 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v40, v[0:3], s[0:1] sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: global_store_dwordx4 v40, v[4:7], s[0:1] offset:16 sc0 sc1
; VGPR-NEXT: s_waitcnt vmcnt(0)
; VGPR-NEXT: s_endpgm
%result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 1, i32 2, i32 3)
store volatile <16 x i32> %arg2, ptr addrspace(1) %out
store volatile <16 x i32> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
; SDAG-NEXT: v_mov_b32_e32 v19, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v20, s24
; SDAG-NEXT: v_mov_b32_e32 v21, s25
; SDAG-NEXT: v_mov_b32_e32 v22, s26
; SDAG-NEXT: v_mov_b32_e32 v23, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 7
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
; GISEL-NEXT: v_mov_b32_e32 v16, 0
; GISEL-NEXT: s_nop 7
; GISEL-NEXT: s_nop 2
; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
; HEURRC-NEXT: v_mov_b32_e32 v18, s22
; HEURRC-NEXT: v_mov_b32_e32 v19, s23
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v20, s24
; HEURRC-NEXT: v_mov_b32_e32 v21, s25
; HEURRC-NEXT: v_mov_b32_e32 v22, s26
; HEURRC-NEXT: v_mov_b32_e32 v23, s27
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; HEURRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; HEURRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; HEURRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b32_e32 v4, s24
; AGPR-NEXT: v_mov_b32_e32 v5, s25
; AGPR-NEXT: v_mov_b32_e32 v6, s26
; AGPR-NEXT: v_mov_b32_e32 v7, s27
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15]
; AGPR-NEXT: v_mov_b32_e32 v0, 0
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: s_nop 2
; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v16, s20
; VGPR-NEXT: v_mov_b32_e32 v17, s21
; VGPR-NEXT: v_mov_b32_e32 v18, s22
; VGPR-NEXT: v_mov_b32_e32 v19, s23
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b32_e32 v20, s24
; VGPR-NEXT: v_mov_b32_e32 v21, s25
; VGPR-NEXT: v_mov_b32_e32 v22, s26
; VGPR-NEXT: v_mov_b32_e32 v23, s27
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15]
; VGPR-NEXT: v_mov_b32_e32 v16, 0
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 2
; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; VGPR-NEXT: s_endpgm
%result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 0, i32 0, i32 0)
store <16 x i32> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__vgprcd_mac_flags(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, ptr addrspace(1) %out) #0 {
; SDAG-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b32_e32 v16, s20
; SDAG-NEXT: v_mov_b32_e32 v17, s21
; SDAG-NEXT: v_mov_b32_e32 v18, s22
; SDAG-NEXT: v_mov_b32_e32 v19, s23
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; SDAG-NEXT: v_mov_b32_e32 v20, s24
; SDAG-NEXT: v_mov_b32_e32 v21, s25
; SDAG-NEXT: v_mov_b32_e32 v22, s26
; SDAG-NEXT: v_mov_b32_e32 v23, s27
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; SDAG-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; SDAG-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; SDAG-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; SDAG-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; SDAG-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; SDAG-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; SDAG-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; SDAG-NEXT: v_mov_b32_e32 v16, 0
; SDAG-NEXT: s_nop 7
; SDAG-NEXT: s_nop 2
; SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_load_dwordx8 s[24:31], s[4:5], 0x24
; GISEL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-NEXT: v_mov_b64_e32 v[16:17], s[24:25]
; GISEL-NEXT: v_mov_b64_e32 v[18:19], s[26:27]
; GISEL-NEXT: v_mov_b64_e32 v[20:21], s[28:29]
; GISEL-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GISEL-NEXT: v_mov_b64_e32 v[22:23], s[30:31]
; GISEL-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GISEL-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GISEL-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GISEL-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; GISEL-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; GISEL-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; GISEL-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; GISEL-NEXT: s_nop 1
; GISEL-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; GISEL-NEXT: v_mov_b32_e32 v16, 0
; GISEL-NEXT: s_nop 7
; GISEL-NEXT: s_nop 2
; GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; GISEL-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; HEURRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b32_e32 v16, s20
; HEURRC-NEXT: v_mov_b32_e32 v17, s21
; HEURRC-NEXT: v_mov_b32_e32 v18, s22
; HEURRC-NEXT: v_mov_b32_e32 v19, s23
; HEURRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; HEURRC-NEXT: v_mov_b32_e32 v20, s24
; HEURRC-NEXT: v_mov_b32_e32 v21, s25
; HEURRC-NEXT: v_mov_b32_e32 v22, s26
; HEURRC-NEXT: v_mov_b32_e32 v23, s27
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; HEURRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; HEURRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: v_mov_b32_e32 v16, 0
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: s_nop 2
; HEURRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; HEURRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; HEURRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; HEURRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPRRC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b32_e32 v16, s20
; VGPRRC-NEXT: v_mov_b32_e32 v17, s21
; VGPRRC-NEXT: v_mov_b32_e32 v18, s22
; VGPRRC-NEXT: v_mov_b32_e32 v19, s23
; VGPRRC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPRRC-NEXT: v_mov_b32_e32 v20, s24
; VGPRRC-NEXT: v_mov_b32_e32 v21, s25
; VGPRRC-NEXT: v_mov_b32_e32 v22, s26
; VGPRRC-NEXT: v_mov_b32_e32 v23, s27
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPRRC-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPRRC-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: v_mov_b32_e32 v16, 0
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: s_nop 2
; VGPRRC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; VGPRRC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; VGPRRC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; VGPRRC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; AGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b32_e32 v0, s20
; AGPR-NEXT: v_mov_b32_e32 v1, s21
; AGPR-NEXT: v_mov_b32_e32 v2, s22
; AGPR-NEXT: v_mov_b32_e32 v3, s23
; AGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; AGPR-NEXT: v_mov_b32_e32 v4, s24
; AGPR-NEXT: v_mov_b32_e32 v5, s25
; AGPR-NEXT: v_mov_b32_e32 v6, s26
; AGPR-NEXT: v_mov_b32_e32 v7, s27
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, s8
; AGPR-NEXT: v_accvgpr_write_b32 a1, s9
; AGPR-NEXT: v_accvgpr_write_b32 a2, s10
; AGPR-NEXT: v_accvgpr_write_b32 a3, s11
; AGPR-NEXT: v_accvgpr_write_b32 a4, s12
; AGPR-NEXT: v_accvgpr_write_b32 a5, s13
; AGPR-NEXT: v_accvgpr_write_b32 a6, s14
; AGPR-NEXT: v_accvgpr_write_b32 a7, s15
; AGPR-NEXT: v_accvgpr_write_b32 a8, s16
; AGPR-NEXT: v_accvgpr_write_b32 a9, s17
; AGPR-NEXT: v_accvgpr_write_b32 a10, s18
; AGPR-NEXT: v_accvgpr_write_b32 a11, s19
; AGPR-NEXT: v_accvgpr_write_b32 a12, s20
; AGPR-NEXT: v_accvgpr_write_b32 a13, s21
; AGPR-NEXT: v_accvgpr_write_b32 a14, s22
; AGPR-NEXT: v_accvgpr_write_b32 a15, s23
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] cbsz:3 abid:2 blgp:1
; AGPR-NEXT: v_mov_b32_e32 v0, 0
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: s_nop 2
; AGPR-NEXT: global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
; AGPR-NEXT: global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
; AGPR-NEXT: global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; AGPR-NEXT: global_store_dwordx4 v0, a[0:3], s[0:1]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_i32_32x32x32_i8__vgprcd_mac_flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[20:27], s[4:5], 0x24
; VGPR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xa4
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b32_e32 v16, s20
; VGPR-NEXT: v_mov_b32_e32 v17, s21
; VGPR-NEXT: v_mov_b32_e32 v18, s22
; VGPR-NEXT: v_mov_b32_e32 v19, s23
; VGPR-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
; VGPR-NEXT: v_mov_b32_e32 v20, s24
; VGPR-NEXT: v_mov_b32_e32 v21, s25
; VGPR-NEXT: v_mov_b32_e32 v22, s26
; VGPR-NEXT: v_mov_b32_e32 v23, s27
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[16:17]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[18:19]
; VGPR-NEXT: v_mov_b64_e32 v[12:13], s[20:21]
; VGPR-NEXT: v_mov_b64_e32 v[14:15], s[22:23]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_i32_32x32x32_i8 v[0:15], v[16:19], v[20:23], v[0:15] cbsz:3 abid:2 blgp:1
; VGPR-NEXT: v_mov_b32_e32 v16, 0
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: s_nop 2
; VGPR-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
; VGPR-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
; VGPR-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
; VGPR-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
; VGPR-NEXT: s_endpgm
%result = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %arg2, i32 3, i32 2, i32 1)
store <16 x i32> %result, ptr addrspace(1) %out
ret void
}
; --------------------------------------------------------------------
; llvm.amdgcn.mfma.f32.16x16x32.bf16
; --------------------------------------------------------------------
declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat>, <8 x bfloat>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
define <4 x float> @test_mfma_f32_16x16x32_bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) {
; GCN-LABEL: test_mfma_f32_16x16x32_bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
; GCN-NEXT: s_nop 7
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_f32_16x16x32_bf16:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_f32_16x16x32_bf16:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)
ret <4 x float> %result
}
define <4 x float> @test_mfma_f32_16x16x32_bf16__flags(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) {
; GCN-LABEL: test_mfma_f32_16x16x32_bf16__flags:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_accvgpr_write_b32 a0, v8
; GCN-NEXT: v_accvgpr_write_b32 a1, v9
; GCN-NEXT: v_accvgpr_write_b32 a2, v10
; GCN-NEXT: v_accvgpr_write_b32 a3, v11
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; GCN-NEXT: s_nop 7
; GCN-NEXT: v_accvgpr_read_b32 v0, a0
; GCN-NEXT: v_accvgpr_read_b32 v1, a1
; GCN-NEXT: v_accvgpr_read_b32 v2, a2
; GCN-NEXT: v_accvgpr_read_b32 v3, a3
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; HEURRC-NEXT: v_accvgpr_write_b32 a0, v8
; HEURRC-NEXT: v_accvgpr_write_b32 a1, v9
; HEURRC-NEXT: v_accvgpr_write_b32 a2, v10
; HEURRC-NEXT: v_accvgpr_write_b32 a3, v11
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: v_accvgpr_read_b32 v0, a0
; HEURRC-NEXT: v_accvgpr_read_b32 v1, a1
; HEURRC-NEXT: v_accvgpr_read_b32 v2, a2
; HEURRC-NEXT: v_accvgpr_read_b32 v3, a3
; HEURRC-NEXT: s_setpc_b64 s[30:31]
;
; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1
; VGPRRC-NEXT: s_setpc_b64 s[30:31]
; AGPR-LABEL: test_mfma_f32_16x16x32_bf16__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; AGPR-NEXT: v_accvgpr_write_b32 a0, v8
; AGPR-NEXT: v_accvgpr_write_b32 a1, v9
; AGPR-NEXT: v_accvgpr_write_b32 a2, v10
; AGPR-NEXT: v_accvgpr_write_b32 a3, v11
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:1 abid:1 blgp:1
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: v_accvgpr_read_b32 v0, a0
; AGPR-NEXT: v_accvgpr_read_b32 v1, a1
; AGPR-NEXT: v_accvgpr_read_b32 v2, a2
; AGPR-NEXT: v_accvgpr_read_b32 v3, a3
; AGPR-NEXT: s_setpc_b64 s[30:31]
; VGPR-LABEL: test_mfma_f32_16x16x32_bf16__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:1 abid:1 blgp:1
; VGPR-NEXT: s_setpc_b64 s[30:31]
%result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 1, i32 1, i32 1)
ret <4 x float> %result
}
define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd(ptr addrspace(1) %out, <8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) #0 {
; GCN-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v12, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GCN-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GCN-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GCN-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GCN-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GCN-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
; GCN-NEXT: s_nop 7
; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; GCN-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; AGPR-NEXT: v_mov_b32_e32 v8, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; AGPR-NEXT: v_accvgpr_write_b32 a0, s0
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; AGPR-NEXT: v_accvgpr_write_b32 a1, s1
; AGPR-NEXT: v_accvgpr_write_b32 a2, s2
; AGPR-NEXT: v_accvgpr_write_b32 a3, s3
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3]
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPR-NEXT: v_mov_b32_e32 v12, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11]
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPR-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 0, i32 0, i32 0)
store <4 x float> %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags(ptr addrspace(1) %out, <8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2) #0 {
; GCN-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v12, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; GCN-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; GCN-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; GCN-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; GCN-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; GCN-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; GCN-NEXT: s_nop 1
; GCN-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; GCN-NEXT: s_nop 7
; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; GCN-NEXT: s_endpgm
;
; HEURRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
; HEURRC: ; %bb.0:
; HEURRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; HEURRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; HEURRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; HEURRC-NEXT: v_mov_b32_e32 v12, 0
; HEURRC-NEXT: s_waitcnt lgkmcnt(0)
; HEURRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; HEURRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; HEURRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; HEURRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; HEURRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; HEURRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; HEURRC-NEXT: s_nop 1
; HEURRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; HEURRC-NEXT: s_nop 7
; HEURRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; HEURRC-NEXT: s_endpgm
;
; VGPRRC-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
; VGPRRC: ; %bb.0:
; VGPRRC-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPRRC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPRRC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPRRC-NEXT: v_mov_b32_e32 v12, 0
; VGPRRC-NEXT: s_waitcnt lgkmcnt(0)
; VGPRRC-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPRRC-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPRRC-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPRRC-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; VGPRRC-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPRRC-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; VGPRRC-NEXT: s_nop 1
; VGPRRC-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; VGPRRC-NEXT: s_nop 7
; VGPRRC-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPRRC-NEXT: s_endpgm
; AGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
; AGPR: ; %bb.0:
; AGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; AGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; AGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; AGPR-NEXT: v_mov_b32_e32 v8, 0
; AGPR-NEXT: s_waitcnt lgkmcnt(0)
; AGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; AGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; AGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; AGPR-NEXT: v_accvgpr_write_b32 a0, s0
; AGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; AGPR-NEXT: v_accvgpr_write_b32 a1, s1
; AGPR-NEXT: v_accvgpr_write_b32 a2, s2
; AGPR-NEXT: v_accvgpr_write_b32 a3, s3
; AGPR-NEXT: s_nop 1
; AGPR-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:3 abid:2 blgp:1
; AGPR-NEXT: s_nop 7
; AGPR-NEXT: global_store_dwordx4 v8, a[0:3], s[6:7]
; AGPR-NEXT: s_endpgm
; VGPR-LABEL: test_mfma_f32_16x16x32_bf16_no_agpr__vgprcd__flags:
; VGPR: ; %bb.0:
; VGPR-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
; VGPR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x54
; VGPR-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; VGPR-NEXT: v_mov_b32_e32 v12, 0
; VGPR-NEXT: s_waitcnt lgkmcnt(0)
; VGPR-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
; VGPR-NEXT: v_mov_b64_e32 v[2:3], s[10:11]
; VGPR-NEXT: v_mov_b64_e32 v[4:5], s[12:13]
; VGPR-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
; VGPR-NEXT: v_mov_b64_e32 v[6:7], s[14:15]
; VGPR-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
; VGPR-NEXT: s_nop 1
; VGPR-NEXT: v_mfma_f32_16x16x32_bf16 v[0:3], v[0:3], v[4:7], v[8:11] cbsz:3 abid:2 blgp:1
; VGPR-NEXT: s_nop 7
; VGPR-NEXT: global_store_dwordx4 v12, v[0:3], s[6:7]
; VGPR-NEXT: s_endpgm
%result = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %arg2, i32 3, i32 2, i32 1)
store <4 x float> %result, ptr addrspace(1) %out
ret void
}
attributes #0 = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-agpr-alloc"="0,0" }
attributes #1 = { "amdgpu-flat-work-group-size"="1,64" }