llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX90A-VGPR %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefixes=VGPR,GFX942-VGPR %s

declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16.1k(<4 x i16>, <4 x i16>, <32 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x4bf16.1k(<4 x i16>, <4 x i16>, <16 x float>, i32, i32, i32)
declare <4 x float> @llvm.amdgcn.mfma.f32.4x4x4bf16.1k(<4 x i16>, <4 x i16>, <4 x float>, i32, i32, i32)
declare <16 x float> @llvm.amdgcn.mfma.f32.32x32x8bf16.1k(<4 x i16>, <4 x i16>, <16 x float>, i32, i32, i32)
declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16>, <4 x i16>, <4 x float>, i32, i32, i32)
declare <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double, double, <4 x double>, i32, i32, i32)
declare double @llvm.amdgcn.mfma.f64.4x4x4f64(double, double, double, i32, i32, i32)
declare i32 @llvm.amdgcn.workitem.id.x()

define amdgpu_kernel void @test_mfma_f32_32x32x4bf16_1k(ptr addrspace(1) %arg) #0 {
; GFX90A-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx2 s[34:35], s[4:5], 0x24
; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
; GFX90A-NEXT:    v_mov_b32_e32 v2, 1
; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-NEXT:    v_mov_b32_e32 v0, 2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x0
; GFX90A-NEXT:    s_load_dwordx16 s[0:15], s[34:35], 0x40
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, s16
; GFX90A-NEXT:    v_accvgpr_write_b32 a1, s17
; GFX90A-NEXT:    v_accvgpr_write_b32 a2, s18
; GFX90A-NEXT:    v_accvgpr_write_b32 a3, s19
; GFX90A-NEXT:    v_accvgpr_write_b32 a4, s20
; GFX90A-NEXT:    v_accvgpr_write_b32 a5, s21
; GFX90A-NEXT:    v_accvgpr_write_b32 a6, s22
; GFX90A-NEXT:    v_accvgpr_write_b32 a7, s23
; GFX90A-NEXT:    v_accvgpr_write_b32 a8, s24
; GFX90A-NEXT:    v_accvgpr_write_b32 a9, s25
; GFX90A-NEXT:    v_accvgpr_write_b32 a10, s26
; GFX90A-NEXT:    v_accvgpr_write_b32 a11, s27
; GFX90A-NEXT:    v_accvgpr_write_b32 a12, s28
; GFX90A-NEXT:    v_accvgpr_write_b32 a13, s29
; GFX90A-NEXT:    v_accvgpr_write_b32 a14, s30
; GFX90A-NEXT:    v_accvgpr_write_b32 a15, s31
; GFX90A-NEXT:    v_accvgpr_write_b32 a16, s0
; GFX90A-NEXT:    v_accvgpr_write_b32 a17, s1
; GFX90A-NEXT:    v_accvgpr_write_b32 a18, s2
; GFX90A-NEXT:    v_accvgpr_write_b32 a19, s3
; GFX90A-NEXT:    v_accvgpr_write_b32 a20, s4
; GFX90A-NEXT:    v_accvgpr_write_b32 a21, s5
; GFX90A-NEXT:    v_accvgpr_write_b32 a22, s6
; GFX90A-NEXT:    v_accvgpr_write_b32 a23, s7
; GFX90A-NEXT:    v_accvgpr_write_b32 a24, s8
; GFX90A-NEXT:    v_accvgpr_write_b32 a25, s9
; GFX90A-NEXT:    v_accvgpr_write_b32 a26, s10
; GFX90A-NEXT:    v_accvgpr_write_b32 a27, s11
; GFX90A-NEXT:    v_accvgpr_write_b32 a28, s12
; GFX90A-NEXT:    v_accvgpr_write_b32 a29, s13
; GFX90A-NEXT:    v_accvgpr_write_b32 a30, s14
; GFX90A-NEXT:    v_accvgpr_write_b32 a31, s15
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f32_32x32x4bf16_1k a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 2
; GFX90A-NEXT:    global_store_dwordx4 v1, a[24:27], s[34:35] offset:96
; GFX90A-NEXT:    global_store_dwordx4 v1, a[28:31], s[34:35] offset:112
; GFX90A-NEXT:    global_store_dwordx4 v1, a[16:19], s[34:35] offset:64
; GFX90A-NEXT:    global_store_dwordx4 v1, a[20:23], s[34:35] offset:80
; GFX90A-NEXT:    global_store_dwordx4 v1, a[8:11], s[34:35] offset:32
; GFX90A-NEXT:    global_store_dwordx4 v1, a[12:15], s[34:35] offset:48
; GFX90A-NEXT:    global_store_dwordx4 v1, a[0:3], s[34:35]
; GFX90A-NEXT:    global_store_dwordx4 v1, a[4:7], s[34:35] offset:16
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx2 s[34:35], s[4:5], 0x24
; GFX942-NEXT:    v_mov_b32_e32 v1, 0
; GFX942-NEXT:    v_mov_b32_e32 v2, 1
; GFX942-NEXT:    v_mov_b32_e32 v3, v1
; GFX942-NEXT:    v_mov_b32_e32 v0, 2
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x0
; GFX942-NEXT:    s_load_dwordx16 s[0:15], s[34:35], 0x40
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_accvgpr_write_b32 a0, s16
; GFX942-NEXT:    v_accvgpr_write_b32 a1, s17
; GFX942-NEXT:    v_accvgpr_write_b32 a2, s18
; GFX942-NEXT:    v_accvgpr_write_b32 a3, s19
; GFX942-NEXT:    v_accvgpr_write_b32 a4, s20
; GFX942-NEXT:    v_accvgpr_write_b32 a5, s21
; GFX942-NEXT:    v_accvgpr_write_b32 a6, s22
; GFX942-NEXT:    v_accvgpr_write_b32 a7, s23
; GFX942-NEXT:    v_accvgpr_write_b32 a8, s24
; GFX942-NEXT:    v_accvgpr_write_b32 a9, s25
; GFX942-NEXT:    v_accvgpr_write_b32 a10, s26
; GFX942-NEXT:    v_accvgpr_write_b32 a11, s27
; GFX942-NEXT:    v_accvgpr_write_b32 a12, s28
; GFX942-NEXT:    v_accvgpr_write_b32 a13, s29
; GFX942-NEXT:    v_accvgpr_write_b32 a14, s30
; GFX942-NEXT:    v_accvgpr_write_b32 a15, s31
; GFX942-NEXT:    v_accvgpr_write_b32 a16, s0
; GFX942-NEXT:    v_accvgpr_write_b32 a17, s1
; GFX942-NEXT:    v_accvgpr_write_b32 a18, s2
; GFX942-NEXT:    v_accvgpr_write_b32 a19, s3
; GFX942-NEXT:    v_accvgpr_write_b32 a20, s4
; GFX942-NEXT:    v_accvgpr_write_b32 a21, s5
; GFX942-NEXT:    v_accvgpr_write_b32 a22, s6
; GFX942-NEXT:    v_accvgpr_write_b32 a23, s7
; GFX942-NEXT:    v_accvgpr_write_b32 a24, s8
; GFX942-NEXT:    v_accvgpr_write_b32 a25, s9
; GFX942-NEXT:    v_accvgpr_write_b32 a26, s10
; GFX942-NEXT:    v_accvgpr_write_b32 a27, s11
; GFX942-NEXT:    v_accvgpr_write_b32 a28, s12
; GFX942-NEXT:    v_accvgpr_write_b32 a29, s13
; GFX942-NEXT:    v_accvgpr_write_b32 a30, s14
; GFX942-NEXT:    v_accvgpr_write_b32 a31, s15
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f32_32x32x4_2b_bf16 a[0:31], v[2:3], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 2
; GFX942-NEXT:    global_store_dwordx4 v1, a[24:27], s[34:35] offset:96
; GFX942-NEXT:    global_store_dwordx4 v1, a[28:31], s[34:35] offset:112
; GFX942-NEXT:    global_store_dwordx4 v1, a[16:19], s[34:35] offset:64
; GFX942-NEXT:    global_store_dwordx4 v1, a[20:23], s[34:35] offset:80
; GFX942-NEXT:    global_store_dwordx4 v1, a[8:11], s[34:35] offset:32
; GFX942-NEXT:    global_store_dwordx4 v1, a[12:15], s[34:35] offset:48
; GFX942-NEXT:    global_store_dwordx4 v1, a[0:3], s[34:35]
; GFX942-NEXT:    global_store_dwordx4 v1, a[4:7], s[34:35] offset:16
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[34:35], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v33, 0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v34, 1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v35, v33
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v32, 2
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x0
; GFX90A-VGPR-NEXT:    s_load_dwordx16 s[0:15], s[34:35], 0x40
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v0, s16
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v1, s17
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v2, s18
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v3, s19
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v4, s20
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v5, s21
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v6, s22
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v7, s23
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, s24
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v9, s25
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v10, s26
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v11, s27
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v12, s28
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v13, s29
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v14, s30
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v15, s31
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v16, s0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v17, s1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v18, s2
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v19, s3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v20, s4
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v21, s5
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v22, s6
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v23, s7
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v24, s8
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v25, s9
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v26, s10
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v27, s11
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v28, s12
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v29, s13
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v30, s14
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v31, s15
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f32_32x32x4bf16_1k v[0:31], v[34:35], v[32:33], v[0:31] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 2
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v33, v[24:27], s[34:35] offset:96
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v33, v[28:31], s[34:35] offset:112
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v33, v[16:19], s[34:35] offset:64
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v33, v[20:23], s[34:35] offset:80
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v33, v[8:11], s[34:35] offset:32
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v33, v[12:15], s[34:35] offset:48
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v33, v[0:3], s[34:35]
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x4bf16_1k:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[34:35], s[4:5], 0x24
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v33, 0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v34, 1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v35, v33
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v32, 2
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    s_load_dwordx16 s[16:31], s[34:35], 0x0
; GFX942-VGPR-NEXT:    s_load_dwordx16 s[0:15], s[34:35], 0x40
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v0, s16
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v1, s17
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v2, s18
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v3, s19
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v4, s20
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v5, s21
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v6, s22
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v7, s23
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, s24
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v9, s25
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v10, s26
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v11, s27
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v12, s28
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v13, s29
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v14, s30
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v15, s31
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v16, s0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v17, s1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v18, s2
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v19, s3
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v20, s4
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v21, s5
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v22, s6
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v23, s7
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v24, s8
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v25, s9
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v26, s10
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v27, s11
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v28, s12
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v29, s13
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v30, s14
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v31, s15
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f32_32x32x4_2b_bf16 v[0:31], v[34:35], v[32:33], v[0:31] cbsz:1 abid:2 blgp:3
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 2
; GFX942-VGPR-NEXT:    global_store_dwordx4 v33, v[24:27], s[34:35] offset:96
; GFX942-VGPR-NEXT:    global_store_dwordx4 v33, v[28:31], s[34:35] offset:112
; GFX942-VGPR-NEXT:    global_store_dwordx4 v33, v[16:19], s[34:35] offset:64
; GFX942-VGPR-NEXT:    global_store_dwordx4 v33, v[20:23], s[34:35] offset:80
; GFX942-VGPR-NEXT:    global_store_dwordx4 v33, v[8:11], s[34:35] offset:32
; GFX942-VGPR-NEXT:    global_store_dwordx4 v33, v[12:15], s[34:35] offset:48
; GFX942-VGPR-NEXT:    global_store_dwordx4 v33, v[0:3], s[34:35]
; GFX942-VGPR-NEXT:    global_store_dwordx4 v33, v[4:7], s[34:35] offset:16
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %in.1 = load <32 x float>, ptr addrspace(1) %arg
  %a = bitcast i64 1 to <4 x i16>
  %b = bitcast i64 2 to <4 x i16>
  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16.1k(<4 x i16> %a, <4 x i16> %b, <32 x float> %in.1, i32 1, i32 2, i32 3)
  store <32 x float> %mai.1, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f32_16x16x4bf16_1k(ptr addrspace(1) %arg) #0 {
; GFX90A-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
; GFX90A-NEXT:    v_mov_b32_e32 v2, 1
; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-NEXT:    v_mov_b32_e32 v0, 2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX90A-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX90A-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX90A-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX90A-NEXT:    v_accvgpr_write_b32 a4, s4
; GFX90A-NEXT:    v_accvgpr_write_b32 a5, s5
; GFX90A-NEXT:    v_accvgpr_write_b32 a6, s6
; GFX90A-NEXT:    v_accvgpr_write_b32 a7, s7
; GFX90A-NEXT:    v_accvgpr_write_b32 a8, s8
; GFX90A-NEXT:    v_accvgpr_write_b32 a9, s9
; GFX90A-NEXT:    v_accvgpr_write_b32 a10, s10
; GFX90A-NEXT:    v_accvgpr_write_b32 a11, s11
; GFX90A-NEXT:    v_accvgpr_write_b32 a12, s12
; GFX90A-NEXT:    v_accvgpr_write_b32 a13, s13
; GFX90A-NEXT:    v_accvgpr_write_b32 a14, s14
; GFX90A-NEXT:    v_accvgpr_write_b32 a15, s15
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f32_16x16x4bf16_1k a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 2
; GFX90A-NEXT:    global_store_dwordx4 v1, a[12:15], s[16:17] offset:48
; GFX90A-NEXT:    global_store_dwordx4 v1, a[8:11], s[16:17] offset:32
; GFX90A-NEXT:    global_store_dwordx4 v1, a[4:7], s[16:17] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v1, a[0:3], s[16:17]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX942-NEXT:    v_mov_b32_e32 v1, 0
; GFX942-NEXT:    v_mov_b32_e32 v2, 1
; GFX942-NEXT:    v_mov_b32_e32 v3, v1
; GFX942-NEXT:    v_mov_b32_e32 v0, 2
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX942-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX942-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX942-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX942-NEXT:    v_accvgpr_write_b32 a4, s4
; GFX942-NEXT:    v_accvgpr_write_b32 a5, s5
; GFX942-NEXT:    v_accvgpr_write_b32 a6, s6
; GFX942-NEXT:    v_accvgpr_write_b32 a7, s7
; GFX942-NEXT:    v_accvgpr_write_b32 a8, s8
; GFX942-NEXT:    v_accvgpr_write_b32 a9, s9
; GFX942-NEXT:    v_accvgpr_write_b32 a10, s10
; GFX942-NEXT:    v_accvgpr_write_b32 a11, s11
; GFX942-NEXT:    v_accvgpr_write_b32 a12, s12
; GFX942-NEXT:    v_accvgpr_write_b32 a13, s13
; GFX942-NEXT:    v_accvgpr_write_b32 a14, s14
; GFX942-NEXT:    v_accvgpr_write_b32 a15, s15
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f32_16x16x4_4b_bf16 a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 2
; GFX942-NEXT:    global_store_dwordx4 v1, a[12:15], s[16:17] offset:48
; GFX942-NEXT:    global_store_dwordx4 v1, a[8:11], s[16:17] offset:32
; GFX942-NEXT:    global_store_dwordx4 v1, a[4:7], s[16:17] offset:16
; GFX942-NEXT:    global_store_dwordx4 v1, a[0:3], s[16:17]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v17, 0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v18, 1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v19, v17
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v16, 2
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f32_16x16x4bf16_1k v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 2
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v17, v[12:15], s[16:17] offset:48
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v17, v[8:11], s[16:17] offset:32
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v17, v[4:7], s[16:17] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x4bf16_1k:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v17, 0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v18, 1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v19, v17
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v16, 2
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f32_16x16x4_4b_bf16 v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 2
; GFX942-VGPR-NEXT:    global_store_dwordx4 v17, v[12:15], s[16:17] offset:48
; GFX942-VGPR-NEXT:    global_store_dwordx4 v17, v[8:11], s[16:17] offset:32
; GFX942-VGPR-NEXT:    global_store_dwordx4 v17, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %in.1 = load <16 x float>, ptr addrspace(1) %arg
  %a = bitcast i64 1 to <4 x i16>
  %b = bitcast i64 2 to <4 x i16>
  %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.16x16x4bf16.1k(<4 x i16> %a, <4 x i16> %b, <16 x float> %in.1, i32 1, i32 2, i32 3)
  store <16 x float> %mai.1, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f32_4x4x4bf16_1k(ptr addrspace(1) %arg) #0 {
; GFX90A-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
; GFX90A-NEXT:    v_mov_b32_e32 v2, 1
; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-NEXT:    v_mov_b32_e32 v0, 2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX90A-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX90A-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX90A-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f32_4x4x4bf16_1k a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    s_nop 4
; GFX90A-NEXT:    global_store_dwordx4 v1, a[0:3], s[6:7]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-NEXT:    v_mov_b32_e32 v1, 0
; GFX942-NEXT:    v_mov_b32_e32 v2, 1
; GFX942-NEXT:    v_mov_b32_e32 v3, v1
; GFX942-NEXT:    v_mov_b32_e32 v0, 2
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX942-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX942-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX942-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f32_4x4x4_16b_bf16 a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3
; GFX942-NEXT:    s_nop 4
; GFX942-NEXT:    global_store_dwordx4 v1, a[0:3], s[6:7]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v5, 0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v6, 1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v7, v5
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v4, 2
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f32_4x4x4bf16_1k v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    s_nop 4
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f32_4x4x4bf16_1k:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v5, 0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v6, 1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v7, v5
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v4, 2
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f32_4x4x4_16b_bf16 v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3
; GFX942-VGPR-NEXT:    s_nop 4
; GFX942-VGPR-NEXT:    global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %in.1 = load <4 x float>, ptr addrspace(1) %arg
  %a = bitcast i64 1 to <4 x i16>
  %b = bitcast i64 2 to <4 x i16>
  %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.4x4x4bf16.1k(<4 x i16> %a, <4 x i16> %b, <4 x float> %in.1, i32 1, i32 2, i32 3)
  store <4 x float> %mai.1, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f32_32x32x8bf16_1k(ptr addrspace(1) %arg) #0 {
; GFX90A-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
; GFX90A-NEXT:    v_mov_b32_e32 v2, 1
; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-NEXT:    v_mov_b32_e32 v0, 2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX90A-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX90A-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX90A-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX90A-NEXT:    v_accvgpr_write_b32 a4, s4
; GFX90A-NEXT:    v_accvgpr_write_b32 a5, s5
; GFX90A-NEXT:    v_accvgpr_write_b32 a6, s6
; GFX90A-NEXT:    v_accvgpr_write_b32 a7, s7
; GFX90A-NEXT:    v_accvgpr_write_b32 a8, s8
; GFX90A-NEXT:    v_accvgpr_write_b32 a9, s9
; GFX90A-NEXT:    v_accvgpr_write_b32 a10, s10
; GFX90A-NEXT:    v_accvgpr_write_b32 a11, s11
; GFX90A-NEXT:    v_accvgpr_write_b32 a12, s12
; GFX90A-NEXT:    v_accvgpr_write_b32 a13, s13
; GFX90A-NEXT:    v_accvgpr_write_b32 a14, s14
; GFX90A-NEXT:    v_accvgpr_write_b32 a15, s15
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f32_32x32x8bf16_1k a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 2
; GFX90A-NEXT:    global_store_dwordx4 v1, a[12:15], s[16:17] offset:48
; GFX90A-NEXT:    global_store_dwordx4 v1, a[8:11], s[16:17] offset:32
; GFX90A-NEXT:    global_store_dwordx4 v1, a[4:7], s[16:17] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v1, a[0:3], s[16:17]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX942-NEXT:    v_mov_b32_e32 v1, 0
; GFX942-NEXT:    v_mov_b32_e32 v2, 1
; GFX942-NEXT:    v_mov_b32_e32 v3, v1
; GFX942-NEXT:    v_mov_b32_e32 v0, 2
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX942-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX942-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX942-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX942-NEXT:    v_accvgpr_write_b32 a4, s4
; GFX942-NEXT:    v_accvgpr_write_b32 a5, s5
; GFX942-NEXT:    v_accvgpr_write_b32 a6, s6
; GFX942-NEXT:    v_accvgpr_write_b32 a7, s7
; GFX942-NEXT:    v_accvgpr_write_b32 a8, s8
; GFX942-NEXT:    v_accvgpr_write_b32 a9, s9
; GFX942-NEXT:    v_accvgpr_write_b32 a10, s10
; GFX942-NEXT:    v_accvgpr_write_b32 a11, s11
; GFX942-NEXT:    v_accvgpr_write_b32 a12, s12
; GFX942-NEXT:    v_accvgpr_write_b32 a13, s13
; GFX942-NEXT:    v_accvgpr_write_b32 a14, s14
; GFX942-NEXT:    v_accvgpr_write_b32 a15, s15
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f32_32x32x8_bf16 a[0:15], v[2:3], v[0:1], a[0:15] cbsz:1 abid:2 blgp:3
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 2
; GFX942-NEXT:    global_store_dwordx4 v1, a[12:15], s[16:17] offset:48
; GFX942-NEXT:    global_store_dwordx4 v1, a[8:11], s[16:17] offset:32
; GFX942-NEXT:    global_store_dwordx4 v1, a[4:7], s[16:17] offset:16
; GFX942-NEXT:    global_store_dwordx4 v1, a[0:3], s[16:17]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v17, 0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v18, 1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v19, v17
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v16, 2
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[8:9], s[8:9] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[10:11], s[10:11] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[12:13], s[12:13], s[12:13] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[14:15], s[14:15], s[14:15] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f32_32x32x8bf16_1k v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 2
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v17, v[12:15], s[16:17] offset:48
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v17, v[8:11], s[16:17] offset:32
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v17, v[4:7], s[16:17] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f32_32x32x8bf16_1k:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x24
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v17, 0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v18, 1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v19, v17
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v16, 2
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f32_32x32x8_bf16 v[0:15], v[18:19], v[16:17], v[0:15] cbsz:1 abid:2 blgp:3
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 2
; GFX942-VGPR-NEXT:    global_store_dwordx4 v17, v[12:15], s[16:17] offset:48
; GFX942-VGPR-NEXT:    global_store_dwordx4 v17, v[8:11], s[16:17] offset:32
; GFX942-VGPR-NEXT:    global_store_dwordx4 v17, v[4:7], s[16:17] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v17, v[0:3], s[16:17]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %in.1 = load <16 x float>, ptr addrspace(1) %arg
  %a = bitcast i64 1 to <4 x i16>
  %b = bitcast i64 2 to <4 x i16>
  %mai.1 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8bf16.1k(<4 x i16> %a, <4 x i16> %b, <16 x float> %in.1, i32 1, i32 2, i32 3)
  store <16 x float> %mai.1, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f32_16x16x16bf16_1k(ptr addrspace(1) %arg) #0 {
; GFX90A-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
; GFX90A-NEXT:    v_mov_b32_e32 v2, 1
; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-NEXT:    v_mov_b32_e32 v0, 2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX90A-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX90A-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX90A-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f32_16x16x16bf16_1k a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 2
; GFX90A-NEXT:    global_store_dwordx4 v1, a[0:3], s[6:7]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-NEXT:    v_mov_b32_e32 v1, 0
; GFX942-NEXT:    v_mov_b32_e32 v2, 1
; GFX942-NEXT:    v_mov_b32_e32 v3, v1
; GFX942-NEXT:    v_mov_b32_e32 v0, 2
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX942-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX942-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX942-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f32_16x16x16_bf16 a[0:3], v[2:3], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3
; GFX942-NEXT:    s_nop 6
; GFX942-NEXT:    global_store_dwordx4 v1, a[0:3], s[6:7]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v5, 0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v6, 1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v7, v5
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v4, 2
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f32_16x16x16bf16_1k v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 2
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f32_16x16x16bf16_1k:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v5, 0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v6, 1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v7, v5
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v4, 2
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[6:7], 0x0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f32_16x16x16_bf16 v[0:3], v[6:7], v[4:5], v[0:3] cbsz:1 abid:2 blgp:3
; GFX942-VGPR-NEXT:    s_nop 6
; GFX942-VGPR-NEXT:    global_store_dwordx4 v5, v[0:3], s[6:7]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %in.1 = load <4 x float>, ptr addrspace(1) %arg
  %a = bitcast i64 1 to <4 x i16>
  %b = bitcast i64 2 to <4 x i16>
  %mai.1 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %a, <4 x i16> %b, <4 x float> %in.1, i32 1, i32 2, i32 3)
  store <4 x float> %mai.1, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_4x4x4f64(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_4x4x4f64:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], 0
; GFX90A-NEXT:    s_nop 3
; GFX90A-NEXT:    v_mfma_f64_4x4x4f64 a[0:1], v[0:1], v[2:3], a[0:1] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    global_store_dwordx2 v0, a[0:1], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_4x4x4f64:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], v[2:3], 0
; GFX942-NEXT:    s_nop 3
; GFX942-NEXT:    v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], v[2:3], a[0:1] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    global_store_dwordx2 v0, a[0:1], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_4x4x4f64:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_4x4x4f64 v[4:5], v[0:1], v[2:3], 0
; GFX90A-VGPR-NEXT:    s_nop 3
; GFX90A-VGPR-NEXT:    v_mfma_f64_4x4x4f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v2, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_4x4x4f64:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_4x4x4_4b_f64 v[4:5], v[0:1], v[2:3], 0
; GFX942-VGPR-NEXT:    s_nop 3
; GFX942-VGPR-NEXT:    v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[4:5] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v2, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double 0.0, i32 0, i32 0, i32 0)
  %mai.2 = tail call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %a, double %b, double %mai.1, i32 1, i32 2, i32 3)
  store double %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x34
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v2, s10
; GFX90A-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX90A-NEXT:    v_mov_b32_e32 v3, s11
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[12:13], s[12:13] op_sel:[0,1]
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX90A-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX90A-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX90A-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX90A-NEXT:    v_accvgpr_write_b32 a4, s4
; GFX90A-NEXT:    v_accvgpr_write_b32 a5, s5
; GFX90A-NEXT:    v_accvgpr_write_b32 a6, s6
; GFX90A-NEXT:    v_accvgpr_write_b32 a7, s7
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[8:9] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[8:9]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x34
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b32_e32 v2, s10
; GFX942-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX942-NEXT:    v_mov_b32_e32 v3, s11
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[12:13]
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_accvgpr_write_b32 a0, s0
; GFX942-NEXT:    v_accvgpr_write_b32 a1, s1
; GFX942-NEXT:    v_accvgpr_write_b32 a2, s2
; GFX942-NEXT:    v_accvgpr_write_b32 a3, s3
; GFX942-NEXT:    v_accvgpr_write_b32 a4, s4
; GFX942-NEXT:    v_accvgpr_write_b32 a5, s5
; GFX942-NEXT:    v_accvgpr_write_b32 a6, s6
; GFX942-NEXT:    v_accvgpr_write_b32 a7, s7
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[8:9] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[8:9]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v10, s10
; GFX90A-VGPR-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v11, s11
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[12:13], s[12:13] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[4:5], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[6:7], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[10:11], v[8:9], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 0
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x34
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v10, s10
; GFX942-VGPR-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v11, s11
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[12:13]
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[6:7], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[10:11], v[8:9], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 0
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[8:9] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[8:9]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %in.1 = load <4 x double>, ptr addrspace(1) %arg
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %in.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.1, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_0(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 0
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 0
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 0
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 0
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_0:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 0
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 0
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> zeroinitializer, i32 0, i32 0, i32 0)
  %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_neg1(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], -1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], -1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 0
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_neg1:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 0
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 -1 to double)), i32 0, i32 0, i32 0)
  %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_1(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 1.0
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 1.0
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 1.0
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 0
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_1:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 1.0
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 0
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double 1.0), i32 0, i32 0, i32 0)
  %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_neg1(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], -1.0
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], -1.0
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], -1.0
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 0
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_neg1:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], -1.0
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 0
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double -1.0), i32 0, i32 0, i32 0)
  %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], 64
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], 64
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], 64
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 0
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], 64
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 0
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 64 to double)), i32 0, i32 0, i32 0)
  %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, 0
; GFX90A-NEXT:    v_accvgpr_write_b32 a1, 64
; GFX90A-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_accvgpr_mov_b32 a3, a1
; GFX90A-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a5, a1
; GFX90A-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a7, a1
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7]
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    v_accvgpr_write_b32 a0, 0
; GFX942-NEXT:    v_accvgpr_write_b32 a1, 64
; GFX942-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_accvgpr_mov_b32 a3, a1
; GFX942-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a5, a1
; GFX942-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a7, a1
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7]
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v1, 64
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v5, v1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v7, v1
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[12:13], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[2:9], v[10:11], v[12:13], v[2:9]
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[2:9], v[10:11], v[12:13], v[2:9] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_bits:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v1, 64
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v3, v1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v5, v1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v7, v1
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], v[6:7]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[12:13], s[6:7]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[6:7], v[4:5]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], v[0:1]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[2:9], v[10:11], v[12:13], v[2:9]
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[2:9], v[10:11], v[12:13], v[2:9] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877906944 to double)), i32 0, i32 0, i32 0)
  %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, 64
; GFX90A-NEXT:    v_accvgpr_mov_b32 a1, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_accvgpr_mov_b32 a3, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a5, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a7, a0
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7]
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    v_accvgpr_write_b32 a0, 64
; GFX942-NEXT:    v_accvgpr_mov_b32 a1, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_accvgpr_mov_b32 a3, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a5, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a7, a0
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7]
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v0, 64
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v1, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v3, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v5, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v7, v0
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 0
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_int_64_in_high_and_low:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v0, 64
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v1, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v3, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v5, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v7, v0
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 0
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (i64 274877907008 to double)), i32 0, i32 0, i32 0)
  %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, 1.0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a1, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT:    v_accvgpr_mov_b32 a3, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a5, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a7, a0
; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7]
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    v_accvgpr_write_b32 a0, 1.0
; GFX942-NEXT:    v_accvgpr_mov_b32 a1, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT:    v_accvgpr_mov_b32 a3, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a5, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a7, a0
; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7]
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[0:1], v[2:3], a[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v1, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v3, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v5, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v7, v0
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7]
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 blgp:3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 0
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_imm_f32_1_in_high_and_low:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v0, 1.0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v1, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], s[2:3]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v3, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v5, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v7, v0
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7]
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[0:7], v[8:9], v[10:11], v[0:7] cbsz:1 abid:2 neg:[1,1,0]
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v8, 0
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 0
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v8, v[0:3], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> splat (double bitcast (<2 x float> splat (float 1.0) to double)), i32 0, i32 0, i32 0)
  %mai.2 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> %mai.1, i32 1, i32 2, i32 3)
  store <4 x double> %mai.2, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_imm(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, 0
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0x3ff00000
; GFX90A-NEXT:    v_accvgpr_write_b32 a7, v0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v2, s2
; GFX90A-NEXT:    v_mov_b32_e32 v3, s3
; GFX90A-NEXT:    v_accvgpr_mov_b32 a1, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a3, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a5, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7]
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    v_accvgpr_write_b32 a0, 0
; GFX942-NEXT:    v_mov_b32_e32 v0, 0x3ff00000
; GFX942-NEXT:    v_accvgpr_write_b32 a7, v0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b32_e32 v2, s2
; GFX942-NEXT:    v_mov_b32_e32 v3, s3
; GFX942-NEXT:    v_accvgpr_mov_b32 a1, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a3, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a5, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v7, 0x3ff00000
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v12, s2
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v13, s3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v3, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v5, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v1, v0
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[2:9], v[12:13], v[10:11], v[2:9]
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_imm:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v7, 0x3ff00000
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v12, s2
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v13, s3
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v3, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v5, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v1, v0
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], v[6:7]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[6:7], v[4:5]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], v[0:1]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[2:9], v[12:13], v[10:11], v[2:9]
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 0.0, double 0.0, double 0.0, double 1.0>, i32 0, i32 0, i32 0)
  store <4 x double> %mai.1, ptr addrspace(1) %arg
  ret void
}

define amdgpu_kernel void @test_mfma_f64_16x16x4f64_splat_lit(ptr addrspace(1) %arg, double %a, double %b) #0 {
; GFX90A-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX90A:       ; %bb.0: ; %bb
; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0x405ec000
; GFX90A-NEXT:    v_accvgpr_write_b32 a0, 0
; GFX90A-NEXT:    v_accvgpr_write_b32 a1, v0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v2, s2
; GFX90A-NEXT:    v_mov_b32_e32 v3, s3
; GFX90A-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a3, a1
; GFX90A-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a5, a1
; GFX90A-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX90A-NEXT:    v_accvgpr_mov_b32 a7, a1
; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-NEXT:    s_nop 1
; GFX90A-NEXT:    v_mfma_f64_16x16x4f64 a[0:7], v[2:3], v[0:1], a[0:7]
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 7
; GFX90A-NEXT:    s_nop 0
; GFX90A-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX90A-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX90A-NEXT:    s_endpgm
;
; GFX942-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX942:       ; %bb.0: ; %bb
; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-NEXT:    v_mov_b32_e32 v0, 0x405ec000
; GFX942-NEXT:    v_accvgpr_write_b32 a0, 0
; GFX942-NEXT:    v_accvgpr_write_b32 a1, v0
; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-NEXT:    v_mov_b32_e32 v2, s2
; GFX942-NEXT:    v_mov_b32_e32 v3, s3
; GFX942-NEXT:    v_accvgpr_mov_b32 a2, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a3, a1
; GFX942-NEXT:    v_accvgpr_mov_b32 a4, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a5, a1
; GFX942-NEXT:    v_accvgpr_mov_b32 a6, a0
; GFX942-NEXT:    v_accvgpr_mov_b32 a7, a1
; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
; GFX942-NEXT:    s_nop 1
; GFX942-NEXT:    v_mfma_f64_16x16x4_f64 a[0:7], v[2:3], v[0:1], a[0:7]
; GFX942-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 7
; GFX942-NEXT:    s_nop 0
; GFX942-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
; GFX942-NEXT:    s_endpgm
;
; GFX90A-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX90A-VGPR:       ; %bb.0: ; %bb
; GFX90A-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v1, 0x405ec000
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX90A-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v12, s2
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v13, s3
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v5, v1
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX90A-VGPR-NEXT:    v_mov_b32_e32 v7, v1
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[10:11], s[6:7], s[6:7] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    v_mfma_f64_16x16x4f64 v[2:9], v[12:13], v[10:11], v[2:9]
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 7
; GFX90A-VGPR-NEXT:    s_nop 1
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX90A-VGPR-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX90A-VGPR-NEXT:    s_endpgm
;
; GFX942-VGPR-LABEL: test_mfma_f64_16x16x4f64_splat_lit:
; GFX942-VGPR:       ; %bb.0: ; %bb
; GFX942-VGPR-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-VGPR-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v0, 0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v1, 0x405ec000
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v2, v0
; GFX942-VGPR-NEXT:    s_waitcnt lgkmcnt(0)
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v12, s2
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v13, s3
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v3, v1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v4, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v5, v1
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v6, v0
; GFX942-VGPR-NEXT:    v_mov_b32_e32 v7, v1
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[8:9], v[6:7]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[10:11], s[6:7]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[6:7], v[4:5]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-VGPR-NEXT:    v_mov_b64_e32 v[2:3], v[0:1]
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    v_mfma_f64_16x16x4_f64 v[2:9], v[12:13], v[10:11], v[2:9]
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 7
; GFX942-VGPR-NEXT:    s_nop 1
; GFX942-VGPR-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
; GFX942-VGPR-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
; GFX942-VGPR-NEXT:    s_endpgm
bb:
  %mai.1 = tail call <4 x double> @llvm.amdgcn.mfma.f64.16x16x4f64(double %a, double %b, <4 x double> <double 123.0, double 123.0, double 123.0, double 123.0>, i32 0, i32 0, i32 0)
  store <4 x double> %mai.1, ptr addrspace(1) %arg
  ret void
}

attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
; VGPR: {{.*}}