Google Git
Sign in
fuchsia / third_party / llvm-project / a336055ddb4290b953871d1714159de4b70670a8 / . / llvm / test / CodeGen / AMDGPU / global-atomics-fp.ll
blob: d10e049444d68541a597b35a94a2cb22f5e1dd8c [file] [log] [blame]
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX900 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s
define amdgpu_kernel void @global_atomic_fadd_ret_f32(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[4:5], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: ; implicit-def: $vgpr1
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT: s_cbranch_execz .LBB0_4
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v5, v1
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB0_2
; GFX900-NEXT: ; %bb.3: ; %Flow
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: .LBB0_4: ; %Flow1
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT: global_store_dword v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB0_4
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB0_2
; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: .LBB0_4: ; %Flow1
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT: global_store_dword v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB0_4
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB0_2
; GFX90A-NEXT: ; %bb.3: ; %Flow
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: .LBB0_4: ; %Flow1
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, exec_lo
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB0_4
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB0_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: .LBB0_4: ; %Flow1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB0_4
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5
; GFX11-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v5, v1
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: s_cbranch_execnz .LBB0_2
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: .LBB0_4: ; %Flow1
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_f32_ieee(ptr addrspace(1) %ptr) #2 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[4:5], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: ; implicit-def: $vgpr1
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT: s_cbranch_execz .LBB1_4
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v5, v1
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB1_2
; GFX900-NEXT: ; %bb.3: ; %Flow
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: .LBB1_4: ; %Flow1
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
; GFX900-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX900-NEXT: v_add_f32_e32 v0, s0, v0
; GFX900-NEXT: global_store_dword v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB1_4
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB1_2
; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: .LBB1_4: ; %Flow1
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
; GFX908-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX908-NEXT: v_add_f32_e32 v0, s0, v0
; GFX908-NEXT: global_store_dword v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB1_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v2, s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: .LBB1_2:
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
; GFX90A-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX90A-NEXT: v_add_f32_e32 v0, s0, v0
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, exec_lo
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB1_4
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB1_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: .LBB1_4: ; %Flow1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX10-NEXT: v_add_f32_e32 v0, s0, v0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB1_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mul_f32 v1, 4.0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v1, v2, v1, s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB1_2:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_noret_f32(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_noret_f32:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[2:3], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT: s_cbranch_execz .LBB2_3
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT: s_mov_b64 s[2:3], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s4
; GFX900-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz .LBB2_2
; GFX900-NEXT: .LBB2_3:
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[2:3], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_cbranch_execz .LBB2_2
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: .LBB2_2:
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB2_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: .LBB2_2:
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s3, exec_lo
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB2_3
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execnz .LBB2_2
; GFX10-NEXT: .LBB2_3:
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB2_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB2_2:
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
ret void
}
define amdgpu_kernel void @global_atomic_fadd_noret_f32_ieee(ptr addrspace(1) %ptr) #2 {
; GFX900-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[2:3], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT: s_cbranch_execz .LBB3_3
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT: s_mov_b64 s[2:3], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s4
; GFX900-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz .LBB3_2
; GFX900-NEXT: .LBB3_3:
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[2:3], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_cbranch_execz .LBB3_2
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: .LBB3_2:
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB3_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: .LBB3_2:
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s3, exec_lo
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB3_3
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execnz .LBB3_2
; GFX10-NEXT: .LBB3_3:
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB3_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB3_2:
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_f32_agent(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[4:5], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: ; implicit-def: $vgpr1
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT: s_cbranch_execz .LBB4_4
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v5, v1
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB4_2
; GFX900-NEXT: ; %bb.3: ; %Flow
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: .LBB4_4: ; %Flow1
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT: global_store_dword v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB4_4
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_2
; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: .LBB4_4: ; %Flow1
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT: global_store_dword v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB4_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v2, s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: .LBB4_2:
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, exec_lo
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB4_4
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB4_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: .LBB4_4: ; %Flow1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB4_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mul_f32 v1, 4.0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v1, v2, v1, s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB4_2:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_f32_system(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_system:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[4:5], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: ; implicit-def: $vgpr1
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT: s_cbranch_execz .LBB5_4
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s6
; GFX900-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v5, v1
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB5_2
; GFX900-NEXT: ; %bb.3: ; %Flow
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: .LBB5_4: ; %Flow1
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT: global_store_dword v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_system:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: ; implicit-def: $vgpr1
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB5_4
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_2
; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: .LBB5_4: ; %Flow1
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT: global_store_dword v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_system:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB5_4
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_2
; GFX90A-NEXT: ; %bb.3: ; %Flow
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: .LBB5_4: ; %Flow1
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_system:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s4, exec_lo
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_4
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s5
; GFX10-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v5, v1
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB5_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: .LBB5_4: ; %Flow1
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT: global_store_dword v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_system:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB5_4
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5
; GFX11-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v5, v1
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: s_cbranch_execnz .LBB5_2
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: .LBB5_4: ; %Flow1
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("one-as") seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT: s_cbranch_execz .LBB6_4
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GCN-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GCN-NEXT: s_mov_b64 s[4:5], 0
; GCN-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_load_dword s6, s[0:1], 0x0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: .LBB6_2: ; %atomicrmw.start
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: v_mov_b32_e32 v5, v1
; GCN-NEXT: v_add_f32_e32 v4, v5, v2
; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1_vol
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GCN-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN-NEXT: s_cbranch_execnz .LBB6_2
; GCN-NEXT: ; %bb.3: ; %Flow
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: .LBB6_4: ; %Flow1
; GCN-NEXT: s_or_b64 exec, exec, s[2:3]
; GCN-NEXT: v_readfirstlane_b32 s0, v1
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GCN-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GCN-NEXT: global_store_dword v[0:1], v0, off
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b64 s[4:5], exec
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX11-NEXT: s_cbranch_execz .LBB6_4
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
; GFX11-NEXT: s_mov_b64 s[4:5], 0
; GFX11-NEXT: v_mul_f32_e32 v2, 4.0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_dword s6, s[0:1], 0x0
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v5, v1
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
; GFX11-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_wbinvl1_vol
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
; GFX11-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX11-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX11-NEXT: s_cbranch_execnz .LBB6_2
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX11-NEXT: .LBB6_4: ; %Flow1
; GFX11-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_mad_f32 v0, v0, 4.0, s0
; GFX11-NEXT: global_store_dword v[0:1], v0, off
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
store float %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[2:3], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB7_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GCN-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1_vol
; GCN-NEXT: .LBB7_2:
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b64 s[2:3], exec
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX11-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX11-NEXT: s_cbranch_execz .LBB7_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: v_mul_f32_e32 v1, 4.0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_wbinvl1_vol
; GFX11-NEXT: .LBB7_2:
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
ret void
}
define amdgpu_kernel void @global_atomic_fadd_noret_f32_safe(ptr addrspace(1) %ptr) {
; GFX900-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[2:3], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT: s_cbranch_execz .LBB8_3
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT: s_mov_b64 s[2:3], 0
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s4
; GFX900-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz .LBB8_2
; GFX900-NEXT: .LBB8_3:
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[2:3], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_cbranch_execz .LBB8_3
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX908-NEXT: s_mov_b64 s[2:3], 0
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX908-NEXT: v_mov_b32_e32 v3, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s4
; GFX908-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_add_f32_e32 v0, v1, v2
; GFX908-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX908-NEXT: s_cbranch_execnz .LBB8_2
; GFX908-NEXT: .LBB8_3:
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB8_3
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_add_f32_e32 v0, v1, v2
; GFX90A-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_2
; GFX90A-NEXT: .LBB8_3:
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s3, exec_lo
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB8_3
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execnz .LBB8_2
; GFX10-NEXT: .LBB8_3:
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: s_mov_b32 s2, 0
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT: s_mov_b32 s4, exec_lo
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB8_3
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4
; GFX11-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_add_f32_e32 v0, v1, v2
; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v0
; GFX11-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
; GFX11-NEXT: s_cbranch_execnz .LBB8_2
; GFX11-NEXT: .LBB8_3:
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
ret void
}
define amdgpu_kernel void @infer_as_before_atomic(ptr addrspace(4) %arg) #0 {
; GFX900-LABEL: infer_as_before_atomic:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_mov_b64 s[2:3], exec
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT: s_cbranch_execz .LBB9_3
; GFX900-NEXT: ; %bb.1:
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT: s_mov_b64 s[2:3], 0
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s4
; GFX900-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: v_mov_b32_e32 v1, v0
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execnz .LBB9_2
; GFX900-NEXT: .LBB9_3:
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: infer_as_before_atomic:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_mov_b64 s[2:3], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_cbranch_execz .LBB9_2
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT: .LBB9_2:
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: infer_as_before_atomic:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB9_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT: .LBB9_2:
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: infer_as_before_atomic:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_mov_b32 s3, exec_lo
; GFX10-NEXT: s_mov_b32 s2, 0
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB9_3
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT: s_cbranch_execnz .LBB9_2
; GFX10-NEXT: .LBB9_3:
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: infer_as_before_atomic:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB9_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
; GFX11-NEXT: .LBB9_2:
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%load = load ptr, ptr addrspace(4) %arg
%v = atomicrmw fadd ptr %load, float 1.0 syncscope("agent-one-as") monotonic, align 4
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX900-NEXT: s_mov_b64 s[0:1], 0
; GFX900-NEXT: s_movk_i32 s4, 0x7fff
; GFX900-NEXT: v_mov_b32_e32 v0, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_and_b32 s2, s6, -4
; GFX900-NEXT: s_mov_b32 s3, s7
; GFX900-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX900-NEXT: s_and_b32 s5, s6, 3
; GFX900-NEXT: s_lshl_b32 s5, s5, 3
; GFX900-NEXT: s_lshl_b32 s6, 0xffff, s5
; GFX900-NEXT: s_not_b32 s6, s6
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s7
; GFX900-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v2, v1
; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX900-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX900-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX900-NEXT: v_add3_u32 v3, v3, v1, s4
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX900-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX900-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX900-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX900-NEXT: s_cbranch_execnz .LBB10_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX900-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX900-NEXT: global_store_short v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX908-NEXT: s_mov_b64 s[0:1], 0
; GFX908-NEXT: s_movk_i32 s4, 0x7fff
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_and_b32 s2, s6, -4
; GFX908-NEXT: s_mov_b32 s3, s7
; GFX908-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX908-NEXT: s_and_b32 s5, s6, 3
; GFX908-NEXT: s_lshl_b32 s5, s5, 3
; GFX908-NEXT: s_lshl_b32 s6, 0xffff, s5
; GFX908-NEXT: s_not_b32 s6, s6
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s7
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v2, v1
; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX908-NEXT: v_add3_u32 v3, v3, v1, s4
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX908-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX908-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX908-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX908-NEXT: global_store_short v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_movk_i32 s4, 0x7fff
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_and_b32 s2, s6, -4
; GFX90A-NEXT: s_mov_b32 s3, s7
; GFX90A-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX90A-NEXT: s_and_b32 s5, s6, 3
; GFX90A-NEXT: s_lshl_b32 s5, s5, 3
; GFX90A-NEXT: s_lshl_b32 s6, 0xffff, s5
; GFX90A-NEXT: s_not_b32 s6, s6
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s7
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX90A-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX90A-NEXT: v_add3_u32 v2, v2, v1, s4
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v1
; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX90A-NEXT: global_store_short v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s0, s2, -4
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_and_b32 s2, s2, 3
; GFX10-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX10-NEXT: s_lshl_b32 s2, s2, 3
; GFX10-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX10-NEXT: s_not_b32 s4, s4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v2, v1
; GFX10-NEXT: v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX10-NEXT: global_store_short v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s0, s2, -4
; GFX11-NEXT: s_mov_b32 s1, s3
; GFX11-NEXT: s_and_b32 s2, s2, 3
; GFX11-NEXT: s_load_b32 s3, s[0:1], 0x0
; GFX11-NEXT: s_lshl_b32 s2, s2, 3
; GFX11-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX11-NEXT: s_not_b32 s4, s4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v2, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v1, s2, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v1, s2, v1
; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX11-NEXT: global_store_b16 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, bfloat 4.0 syncscope("agent") seq_cst
store bfloat %result, ptr addrspace(1) undef
ret void
}
define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX900-NEXT: s_mov_b64 s[0:1], 0
; GFX900-NEXT: s_movk_i32 s4, 0x7fff
; GFX900-NEXT: v_mov_b32_e32 v0, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_and_b32 s2, s6, -4
; GFX900-NEXT: s_mov_b32 s3, s7
; GFX900-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX900-NEXT: s_and_b32 s5, s6, 3
; GFX900-NEXT: s_lshl_b32 s5, s5, 3
; GFX900-NEXT: s_lshl_b32 s6, 0xffff, s5
; GFX900-NEXT: s_not_b32 s6, s6
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, s7
; GFX900-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: v_mov_b32_e32 v2, v1
; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX900-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX900-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX900-NEXT: v_add3_u32 v3, v3, v1, s4
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX900-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX900-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX900-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX900-NEXT: s_cbranch_execnz .LBB11_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX900-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX900-NEXT: global_store_short v[0:1], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX908-NEXT: s_mov_b64 s[0:1], 0
; GFX908-NEXT: s_movk_i32 s4, 0x7fff
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_and_b32 s2, s6, -4
; GFX908-NEXT: s_mov_b32 s3, s7
; GFX908-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX908-NEXT: s_and_b32 s5, s6, 3
; GFX908-NEXT: s_lshl_b32 s5, s5, 3
; GFX908-NEXT: s_lshl_b32 s6, 0xffff, s5
; GFX908-NEXT: s_not_b32 s6, s6
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s7
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_mov_b32_e32 v2, v1
; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX908-NEXT: v_add3_u32 v3, v3, v1, s4
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX908-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX908-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX908-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX908-NEXT: global_store_short v[0:1], v0, off
; GFX908-NEXT: s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_movk_i32 s4, 0x7fff
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_and_b32 s2, s6, -4
; GFX90A-NEXT: s_mov_b32 s3, s7
; GFX90A-NEXT: s_load_dword s7, s[2:3], 0x0
; GFX90A-NEXT: s_and_b32 s5, s6, 3
; GFX90A-NEXT: s_lshl_b32 s5, s5, 3
; GFX90A-NEXT: s_lshl_b32 s6, 0xffff, s5
; GFX90A-NEXT: s_not_b32 s6, s6
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s7
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX90A-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX90A-NEXT: v_add3_u32 v2, v2, v1, s4
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v1
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s5, v1
; GFX90A-NEXT: global_store_short v[0:1], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s0, s2, -4
; GFX10-NEXT: s_mov_b32 s1, s3
; GFX10-NEXT: s_and_b32 s2, s2, 3
; GFX10-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX10-NEXT: s_lshl_b32 s2, s2, 3
; GFX10-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX10-NEXT: s_not_b32 s4, s4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_mov_b32_e32 v2, v1
; GFX10-NEXT: v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: s_cbranch_execnz .LBB11_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX10-NEXT: global_store_short v[0:1], v0, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s0, s2, -4
; GFX11-NEXT: s_mov_b32 s1, s3
; GFX11-NEXT: s_and_b32 s2, s2, 3
; GFX11-NEXT: s_load_b32 s3, s[0:1], 0x0
; GFX11-NEXT: s_lshl_b32 s2, s2, 3
; GFX11-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX11-NEXT: s_not_b32 s4, s4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_mov_b32_e32 v2, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v1, s2, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v1
; GFX11-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-NEXT: v_lshlrev_b32_e32 v1, s2, v1
; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX11-NEXT: global_store_b16 v[0:1], v0, off
; GFX11-NEXT: s_nop 0
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, bfloat 4.0 syncscope("one-as") seq_cst
store bfloat %result, ptr addrspace(1) undef
ret void
}
define <2 x half> @global_atomic_fadd_ret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) {
; GFX900-LABEL: global_atomic_fadd_ret_v2f16:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: global_load_dword v3, v[0:1], off
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v4, v3
; GFX900-NEXT: v_pk_add_f16 v3, v4, v2
; GFX900-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB12_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: v_mov_b32_e32 v0, v3
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_atomic_fadd_ret_v2f16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: v_pk_add_f16 v3, v4, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: global_atomic_fadd_ret_v2f16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2
; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_atomic_fadd_ret_v2f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: v_pk_add_f16 v3, v4, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_atomic_fadd_ret_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
; GFX11-NEXT: v_pk_add_f16 v3, v4, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
ret <2 x half> %result
}
define void @global_atomic_fadd_noret_v2f16(ptr addrspace(1) %ptr, <2 x half> %val) {
; GFX900-LABEL: global_atomic_fadd_noret_v2f16:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: global_load_dword v4, v[0:1], off
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_pk_add_f16 v3, v4, v2
; GFX900-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT: v_mov_b32_e32 v4, v3
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_cbranch_execnz .LBB13_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_atomic_fadd_noret_v2f16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v4, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_pk_add_f16 v3, v4, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: v_mov_b32_e32 v4, v3
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: global_atomic_fadd_noret_v2f16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v5, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2
; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_atomic_fadd_noret_v2f16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v4, v[0:1], off
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_pk_add_f16 v3, v4, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX10-NEXT: v_mov_b32_e32 v4, v3
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB13_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_atomic_fadd_noret_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v4, v[0:1], off
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_pk_add_f16 v3, v4, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-NEXT: v_mov_b32_e32 v4, v3
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, <2 x half> %val syncscope("agent") seq_cst
ret void
}
define <2 x bfloat> @global_atomic_fadd_ret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) {
; GFX900-LABEL: global_atomic_fadd_ret_v2bf16:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: global_load_dword v3, v[0:1], off
; GFX900-NEXT: s_mov_b64 s[6:7], 0
; GFX900-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX900-NEXT: s_movk_i32 s8, 0x7fff
; GFX900-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX900-NEXT: s_mov_b32 s9, 0x7060302
; GFX900-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v6, v3
; GFX900-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX900-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX900-NEXT: v_add_f32_e32 v3, v3, v4
; GFX900-NEXT: v_add_f32_e32 v5, v5, v2
; GFX900-NEXT: v_bfe_u32 v7, v3, 16, 1
; GFX900-NEXT: v_bfe_u32 v9, v5, 16, 1
; GFX900-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX900-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX900-NEXT: v_add3_u32 v7, v7, v3, s8
; GFX900-NEXT: v_add3_u32 v9, v9, v5, s8
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX900-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
; GFX900-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX900-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
; GFX900-NEXT: v_perm_b32 v5, v5, v3, s9
; GFX900-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX900-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX900-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX900-NEXT: s_cbranch_execnz .LBB14_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX900-NEXT: v_mov_b32_e32 v0, v3
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_atomic_fadd_ret_v2bf16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v3
; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: global_atomic_fadd_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v7, v3
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v7
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
; GFX90A-NEXT: v_add_f32_e32 v3, v3, v4
; GFX90A-NEXT: v_add_f32_e32 v5, v5, v2
; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX90A-NEXT: v_bfe_u32 v9, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
; GFX90A-NEXT: v_add3_u32 v9, v9, v5, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v8, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
; GFX90A-NEXT: v_perm_b32 v6, v5, v3, s9
; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[6:7], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v0, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_atomic_fadd_ret_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v3
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB14_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: v_mov_b32_e32 v0, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_atomic_fadd_ret_v2bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v6, v3
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX11-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX11-NEXT: v_add_f32_e32 v3, v3, v4
; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX11-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
; GFX11-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: v_mov_b32_e32 v0, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
ret <2 x bfloat> %result
}
define void @global_atomic_fadd_noret_v2bf16(ptr addrspace(1) %ptr, <2 x bfloat> %val) {
; GFX900-LABEL: global_atomic_fadd_noret_v2bf16:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: global_load_dword v3, v[0:1], off
; GFX900-NEXT: s_mov_b64 s[6:7], 0
; GFX900-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX900-NEXT: s_movk_i32 s8, 0x7fff
; GFX900-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX900-NEXT: s_mov_b32 s9, 0x7060302
; GFX900-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX900-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX900-NEXT: v_add_f32_e32 v2, v2, v4
; GFX900-NEXT: v_add_f32_e32 v6, v6, v5
; GFX900-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX900-NEXT: v_bfe_u32 v9, v6, 16, 1
; GFX900-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX900-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX900-NEXT: v_add3_u32 v7, v7, v2, s8
; GFX900-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX900-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX900-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
; GFX900-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
; GFX900-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
; GFX900-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX900-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: buffer_wbinvl1_vol
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX900-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX900-NEXT: v_mov_b32_e32 v3, v2
; GFX900-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX900-NEXT: s_cbranch_execnz .LBB15_1
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_atomic_fadd_noret_v2bf16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1_vol
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: global_atomic_fadd_noret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_atomic_fadd_noret_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB15_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_atomic_fadd_noret_v2bf16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-NEXT: v_add_f32_e32 v2, v2, v4
; GFX11-NEXT: v_add_f32_e32 v6, v6, v5
; GFX11-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-NEXT: v_bfe_u32 v8, v6, 16, 1
; GFX11-NEXT: v_or_b32_e32 v9, 0x400000, v2
; GFX11-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX11-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
; GFX11-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fadd ptr addrspace(1) %ptr, <2 x bfloat> %val syncscope("agent") seq_cst
ret void
}
attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
attributes #1 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "target-cpu"="gfx803" "target-features"="+atomic-fadd-no-rtn-insts" "amdgpu-unsafe-fp-atomics"="true" }
attributes #2 = { "amdgpu-unsafe-fp-atomics"="true" }